Added tenancy tests for WorkloadHealth controller (#19530)

internal/catalog/internal/controllers/workloadhealth/controller_test.go

@@ -6,17 +6,20 @@ package workloadhealth
 import (
 	"context"
 	"fmt"
-	"github.com/hashicorp/consul/internal/resource"
-	"google.golang.org/protobuf/testing/protocmp"
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/require"
+	"github.com/hashicorp/consul/internal/resource"
-	"github.com/stretchr/testify/suite"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/testing/protocmp"
 
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	svc "github.com/hashicorp/consul/agent/grpc-external/services/resource"
 	svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing"
+	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/internal/catalog/internal/controllers/nodehealth"
 	"github.com/hashicorp/consul/internal/catalog/internal/mappers/nodemapper"
 	"github.com/hashicorp/consul/internal/catalog/internal/types"
@@ -45,10 +48,15 @@ var (
 	}
 )
 
-func resourceID(rtype *pbresource.Type, name string) *pbresource.ID {
+func resourceID(rtype *pbresource.Type, name string, tenancy *pbresource.Tenancy) *pbresource.ID {
+	defaultTenancy := resource.DefaultNamespacedTenancy()
+	if tenancy != nil {
+		defaultTenancy = tenancy
+	}
+
 	return &pbresource.ID{
 		Type:    rtype,
-		Tenancy: resource.DefaultNamespacedTenancy(),
+		Tenancy: defaultTenancy,
 		Name:    name,
 	}
 }
@@ -79,18 +87,31 @@ type controllerSuite struct {
 	suite.Suite
 	client  pbresource.ResourceServiceClient
 	runtime controller.Runtime
+
+	isEnterprise bool
+	tenancies    []*pbresource.Tenancy
 }
 
 func (suite *controllerSuite) SetupTest() {
-	suite.client = svctest.RunResourceService(suite.T(), types.Register)
+	suite.tenancies = resourcetest.TestTenancies()
+	mockTenancyBridge := &svc.MockTenancyBridge{}
+	for _, tenancy := range suite.tenancies {
+		mockTenancyBridge.On("PartitionExists", tenancy.Partition).Return(true, nil)
+		mockTenancyBridge.On("IsPartitionMarkedForDeletion", tenancy.Partition).Return(false, nil)
+		mockTenancyBridge.On("NamespaceExists", tenancy.Partition, tenancy.Namespace).Return(true, nil)
+		mockTenancyBridge.On("IsNamespaceMarkedForDeletion", tenancy.Partition, tenancy.Namespace).Return(false, nil)
+	}
+
+	suite.client = svctest.RunResourceServiceWithConfig(suite.T(), svc.Config{TenancyBridge: mockTenancyBridge}, types.Register)
 	suite.runtime = controller.Runtime{Client: suite.client, Logger: testutil.Logger(suite.T())}
+	suite.isEnterprise = (structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty() == "default")
 }
 
 // injectNodeWithStatus is a helper method to write a Node resource and synthesize its status
 // in a manner consistent with the node-health controller. This allows us to not actually
 // run and test the node-health controller but consume its "api" in the form of how
 // it encodes status.
-func (suite *controllerSuite) injectNodeWithStatus(name string, health pbcatalog.Health) *pbresource.Resource {
+func (suite *controllerSuite) injectNodeWithStatus(name string, health pbcatalog.Health, tenancy *pbresource.Tenancy) *pbresource.Resource {
 	suite.T().Helper()
 	state := pbresource.Condition_STATE_TRUE
 	if health >= pbcatalog.Health_HEALTH_WARNING {
@@ -99,6 +120,7 @@ func (suite *controllerSuite) injectNodeWithStatus(name string, health pbcatalog
 
 	return resourcetest.Resource(pbcatalog.NodeType, name).
 		WithData(suite.T(), nodeData).
+		WithTenancy(tenancy).
 		WithStatus(nodehealth.StatusKey, &pbresource.Status{
 			Conditions: []*pbresource.Condition{
 				{
@@ -142,18 +164,20 @@ func (suite *workloadHealthControllerTestSuite) SetupTest() {
 //
 // * The node to workload association is now being tracked by the node mapper
 // * The workloads status was updated and now matches the expected value
-func (suite *workloadHealthControllerTestSuite) testReconcileWithNode(nodeHealth, workloadHealth pbcatalog.Health, status *pbresource.Condition) *pbresource.Resource {
+func (suite *workloadHealthControllerTestSuite) testReconcileWithNode(nodeHealth, workloadHealth pbcatalog.Health, tenancy *pbresource.Tenancy, status *pbresource.Condition) *pbresource.Resource {
 	suite.T().Helper()
 
-	node := suite.injectNodeWithStatus("test-node", nodeHealth)
+	node := suite.injectNodeWithStatus("test-node", nodeHealth, tenancy)
 
 	workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
 		WithData(suite.T(), workloadData(node.Id.Name)).
+		WithTenancy(tenancy).
 		Write(suite.T(), suite.client)
 
 	resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
 		WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: workloadHealth}).
 		WithOwner(workload.Id).
+		WithTenancy(tenancy).
 		Write(suite.T(), suite.client)
 
 	err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{
@@ -189,14 +213,16 @@ func (suite *workloadHealthControllerTestSuite) testReconcileWithNode(nodeHealth
 // This is really just a tirmmed down version of testReconcileWithNode. It seemed
 // simpler and easier to read if these were two separate methods instead of combining
 // them in one with more branching based off of detecting whether nodes are in use.
-func (suite *workloadHealthControllerTestSuite) testReconcileWithoutNode(workloadHealth pbcatalog.Health, status *pbresource.Condition) *pbresource.Resource {
+func (suite *workloadHealthControllerTestSuite) testReconcileWithoutNode(workloadHealth pbcatalog.Health, tenancy *pbresource.Tenancy, status *pbresource.Condition) *pbresource.Resource {
 	suite.T().Helper()
 	workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
 		WithData(suite.T(), workloadData("")).
+		WithTenancy(tenancy).
 		Write(suite.T(), suite.client)
 
 	resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
 		WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: workloadHealth}).
+		WithTenancy(tenancy).
 		WithOwner(workload.Id).
 		Write(suite.T(), suite.client)
 
@@ -356,11 +382,13 @@ func (suite *workloadHealthControllerTestSuite) TestReconcile() {
 
 	for name, tcase := range cases {
 		suite.Run(name, func() {
-			if tcase.nodeHealth != pbcatalog.Health_HEALTH_ANY {
+			suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-				suite.testReconcileWithNode(tcase.nodeHealth, tcase.workloadHealth, tcase.expectedStatus)
+				if tcase.nodeHealth != pbcatalog.Health_HEALTH_ANY {
-			} else {
+					suite.testReconcileWithNode(tcase.nodeHealth, tcase.workloadHealth, tenancy, tcase.expectedStatus)
-				suite.testReconcileWithoutNode(tcase.workloadHealth, tcase.expectedStatus)
+				} else {
-			}
+					suite.testReconcileWithoutNode(tcase.workloadHealth, tenancy, tcase.expectedStatus)
+				}
+			})
 		})
 	}
 }
@@ -372,56 +400,60 @@ func (suite *workloadHealthControllerTestSuite) TestReconcileReadError() {
 	// Passing a resource with an unknown type isn't particularly realistic as the controller
 	// manager running our reconciliation will ensure all resource ids used are valid. However
 	// its a really easy way right not to force the error.
-	id := resourceID(fakeType, "blah")
+	suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
+		id := resourceID(fakeType, "blah", tenancy)
 
-	err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{ID: id})
+		err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{ID: id})
-	require.Error(suite.T(), err)
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
+		require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
+	})
 }
 
 func (suite *workloadHealthControllerTestSuite) TestReconcileNotFound() {
 	// This test wants to ensure that tracking for a workload is removed when the workload is deleted
 	// so this test will inject the tracking, issue the Reconcile call which will get a
 	// not found error and then ensure that the tracking was removed.
+	suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
+		workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
+			WithData(suite.T(), workloadData("test-node")).
+			// don't write this because then in the call to reconcile the resource
+			// would be found and defeat the purpose of the tes
+			WithTenancy(tenancy).
+			Build()
 
-	workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
+		node := resourcetest.Resource(pbcatalog.NodeType, "test-node").
-		WithData(suite.T(), workloadData("test-node")).
+			WithData(suite.T(), nodeData).
-		// don't write this because then in the call to reconcile the resource
+			WithTenancy(tenancy).
-		// would be found and defeat the purpose of the tes
+			// Whether this gets written or not doesn't matter
-		WithTenancy(resource.DefaultNamespacedTenancy()).
+			Build()
-		Build()
 
-	node := resourcetest.Resource(pbcatalog.NodeType, "test-node").
+		// Track the workload - this simulates a previous round of reconciliation
-		WithData(suite.T(), nodeData).
+		// where the workload existed and was associated to the node. Other tests
-		// Whether this gets written or not doesn't matter
+		// will cover more of the lifecycle of the controller so for the purposes
-		Build()
+		// of this test we can just inject it ourselves.
+		suite.mapper.TrackWorkload(workload.Id, node.Id)
 
-	// Track the workload - this simulates a previous round of reconciliation
+		// check that the worklooad is in fact tracked properly
-	// where the workload existed and was associated to the node. Other tests
+		reqs, err := suite.mapper.MapNodeToWorkloads(context.Background(), suite.runtime, node)
-	// will cover more of the lifecycle of the controller so for the purposes
-	// of this test we can just inject it ourselves.
-	suite.mapper.TrackWorkload(workload.Id, node.Id)
 
-	// check that the worklooad is in fact tracked properly
+		require.NoError(suite.T(), err)
-	reqs, err := suite.mapper.MapNodeToWorkloads(context.Background(), suite.runtime, node)
+		require.Len(suite.T(), reqs, 1)
+		prototest.AssertDeepEqual(suite.T(), workload.Id, reqs[0].ID)
 
-	require.NoError(suite.T(), err)
+		// This workload was never actually inserted so the request should return a NotFound
-	require.Len(suite.T(), reqs, 1)
+		// error and remove the workload from tracking
-	prototest.AssertDeepEqual(suite.T(), workload.Id, reqs[0].ID)
+		require.NoError(
+			suite.T(),
+			suite.reconciler.Reconcile(
+				context.Background(),
+				suite.runtime,
+				controller.Request{ID: workload.Id}))
 
-	// This workload was never actually inserted so the request should return a NotFound
+		// Check the mapper again to ensure the node:workload association was removed.
-	// error and remove the workload from tracking
+		reqs, err = suite.mapper.MapNodeToWorkloads(context.Background(), suite.runtime, node)
-	require.NoError(
+		require.NoError(suite.T(), err)
-		suite.T(),
+		require.Empty(suite.T(), reqs)
-		suite.reconciler.Reconcile(
+	})
-			context.Background(),
-			suite.runtime,
-			controller.Request{ID: workload.Id}))
-
-	// Check the mapper again to ensure the node:workload association was removed.
-	reqs, err = suite.mapper.MapNodeToWorkloads(context.Background(), suite.runtime, node)
-	require.NoError(suite.T(), err)
-	require.Empty(suite.T(), reqs)
 }
 
 func (suite *workloadHealthControllerTestSuite) TestGetNodeHealthError() {
@@ -434,25 +466,30 @@ func (suite *workloadHealthControllerTestSuite) TestGetNodeHealthError() {
 	// but the exact error isn't very relevant to the core reason this
 	// test exists.
 
-	node := resourcetest.Resource(pbcatalog.NodeType, "test-node").
+	suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		WithData(suite.T(), nodeData).
+		node := resourcetest.Resource(pbcatalog.NodeType, "test-node").
-		Write(suite.T(), suite.client)
+			WithData(suite.T(), nodeData).
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
+		workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
-		WithData(suite.T(), workloadData(node.Id.Name)).
+			WithData(suite.T(), workloadData(node.Id.Name)).
-		Write(suite.T(), suite.client)
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
+		resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
-		WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_CRITICAL}).
+			WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_CRITICAL}).
-		WithOwner(workload.Id).
+			WithOwner(workload.Id).
-		Write(suite.T(), suite.client)
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{
+		err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{
-		ID: workload.Id,
+			ID: workload.Id,
+		})
+
+		require.Error(suite.T(), err)
+		require.Equal(suite.T(), errNodeUnreconciled, err)
 	})
-
-	require.Error(suite.T(), err)
-	require.Equal(suite.T(), errNodeUnreconciled, err)
 }
 
 func (suite *workloadHealthControllerTestSuite) TestReconcile_AvoidReconciliationWrite() {
@@ -461,24 +498,26 @@ func (suite *workloadHealthControllerTestSuite) TestReconcile_AvoidReconciliatio
 	// we check that calling Reconcile twice in a row without any actual health change
 	// doesn't bump the Version (which would increased for any write of the resource
 	// or its status)
-	status := &pbresource.Condition{
+	suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		Type:    StatusConditionHealthy,
+		status := &pbresource.Condition{
-		State:   pbresource.Condition_STATE_FALSE,
+			Type:    StatusConditionHealthy,
-		Reason:  "HEALTH_WARNING",
+			State:   pbresource.Condition_STATE_FALSE,
-		Message: WorkloadUnhealthyMessage,
+			Reason:  "HEALTH_WARNING",
-	}
+			Message: WorkloadUnhealthyMessage,
-	res1 := suite.testReconcileWithoutNode(pbcatalog.Health_HEALTH_WARNING, status)
+		}
+		res1 := suite.testReconcileWithoutNode(pbcatalog.Health_HEALTH_WARNING, tenancy, status)
 
-	err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{ID: res1.Id})
+		err := suite.reconciler.Reconcile(context.Background(), suite.runtime, controller.Request{ID: res1.Id})
-	require.NoError(suite.T(), err)
+		require.NoError(suite.T(), err)
 
-	// check that the status hasn't changed
+		// check that the status hasn't changed
-	res2 := suite.checkWorkloadStatus(res1.Id, status)
+		res2 := suite.checkWorkloadStatus(res1.Id, status)
 
-	// If another status write was performed then the versions would differ. This
+		// If another status write was performed then the versions would differ. This
-	// therefore proves that after a second reconciliation without any change
+		// therefore proves that after a second reconciliation without any change
-	// in status that the controller is not making extra status writes.
+		// in status that the controller is not making extra status writes.
-	require.Equal(suite.T(), res1.Version, res2.Version)
+		require.Equal(suite.T(), res1.Version, res2.Version)
+	})
 }
 
 func (suite *workloadHealthControllerTestSuite) TestController() {
@@ -498,47 +537,52 @@ func (suite *workloadHealthControllerTestSuite) TestController() {
 	// run the manager
 	go mgr.Run(ctx)
 
-	// create a node to link things with
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-	node := suite.injectNodeWithStatus("test-node", pbcatalog.Health_HEALTH_PASSING)
+		node := suite.injectNodeWithStatus("test-node", pbcatalog.Health_HEALTH_PASSING, tenancy)
 
-	// create the workload
+		// create the workload
-	workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
+		workload := resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
-		WithData(suite.T(), workloadData(node.Id.Name)).
+			WithData(suite.T(), workloadData(node.Id.Name)).
-		Write(suite.T(), suite.client)
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	// Wait for reconciliation to occur and mark the workload as passing.
+		// Wait for reconciliation to occur and mark the workload as passing.
-	suite.waitForReconciliation(workload.Id, "HEALTH_PASSING")
+		suite.waitForReconciliation(workload.Id, "HEALTH_PASSING")
 
-	// Simulate a node unhealthy
+		// Simulate a node unhealthy
-	suite.injectNodeWithStatus("test-node", pbcatalog.Health_HEALTH_WARNING)
+		suite.injectNodeWithStatus("test-node", pbcatalog.Health_HEALTH_WARNING, tenancy)
 
-	// Wait for reconciliation to occur and mark the workload as warning
+		// Wait for reconciliation to occur and mark the workload as warning
-	// due to the node going into the warning state.
+		// due to the node going into the warning state.
-	suite.waitForReconciliation(workload.Id, "HEALTH_WARNING")
+		suite.waitForReconciliation(workload.Id, "HEALTH_WARNING")
 
-	// Now register a critical health check that should supercede the nodes
+		// Now register a critical health check that should supercede the nodes
-	// warning status
+		// warning status
 
-	resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
+		resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
-		WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_CRITICAL}).
+			WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_CRITICAL}).
-		WithOwner(workload.Id).
+			WithOwner(workload.Id).
-		Write(suite.T(), suite.client)
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	// Wait for reconciliation to occur again and mark the workload as unhealthy
+		// Wait for reconciliation to occur again and mark the workload as unhealthy
-	suite.waitForReconciliation(workload.Id, "HEALTH_CRITICAL")
+		suite.waitForReconciliation(workload.Id, "HEALTH_CRITICAL")
 
-	// Put the health status back into a passing state and delink the node
+		// Put the health status back into a passing state and delink the node
-	resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
+		resourcetest.Resource(pbcatalog.HealthStatusType, "test-status").
-		WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_PASSING}).
+			WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: pbcatalog.Health_HEALTH_PASSING}).
-		WithOwner(workload.Id).
+			WithOwner(workload.Id).
-		Write(suite.T(), suite.client)
+			WithTenancy(tenancy).
-	workload = resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
+			Write(suite.T(), suite.client)
-		WithData(suite.T(), workloadData("")).
+		workload = resourcetest.Resource(pbcatalog.WorkloadType, "test-workload").
-		Write(suite.T(), suite.client)
+			WithData(suite.T(), workloadData("")).
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	// Now that the workload health is passing and its not associated with the node its status should
+		// Now that the workload health is passing and its not associated with the node its status should
-	// eventually become passing
+		// eventually become passing
-	suite.waitForReconciliation(workload.Id, "HEALTH_PASSING")
+		suite.waitForReconciliation(workload.Id, "HEALTH_PASSING")
+	})
 }
 
 // wait for reconciliation is a helper to check if a resource has been reconciled and
@@ -569,7 +613,7 @@ type getWorkloadHealthTestSuite struct {
 	controllerSuite
 }
 
-func (suite *getWorkloadHealthTestSuite) addHealthStatuses(workload *pbresource.ID, desiredHealth pbcatalog.Health) {
+func (suite *getWorkloadHealthTestSuite) addHealthStatuses(workload *pbresource.ID, tenancy *pbresource.Tenancy, desiredHealth pbcatalog.Health) {
 	// In order to exercise the behavior to ensure that the ordering a health status is
 	// seen doesn't matter this is strategically naming health status so that they will be
 	// returned in an order with the most precedent status being in the middle of the list.
@@ -590,6 +634,7 @@ func (suite *getWorkloadHealthTestSuite) addHealthStatuses(workload *pbresource.
 		if desiredHealth >= health {
 			resourcetest.Resource(pbcatalog.HealthStatusType, fmt.Sprintf("check-%s-%d", workload.Name, idx)).
 				WithData(suite.T(), &pbcatalog.HealthStatus{Type: "tcp", Status: health}).
+				WithTenancy(tenancy).
 				WithOwner(workload).
 				Write(suite.T(), suite.client)
 		}
@@ -601,23 +646,28 @@ func (suite *getWorkloadHealthTestSuite) TestListError() {
 	// getWorkloadHealth. When the resource listing fails, we want to
 	// propagate the error which should eventually result in retrying
 	// the operation.
-	health, err := getWorkloadHealth(context.Background(), suite.runtime, resourceID(fakeType, "foo"))
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
+		health, err := getWorkloadHealth(context.Background(), suite.runtime, resourceID(fakeType, "foo", tenancy))
 
-	require.Error(suite.T(), err)
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
+		require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getWorkloadHealthTestSuite) TestNoHealthStatuses() {
 	// This test's goal is to ensure that when no HealthStatuses are owned by the
 	// workload that the health is assumed to be passing.
-	workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		WithData(suite.T(), workloadData("")).
+		workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
-		Write(suite.T(), suite.client)
+			WithData(suite.T(), workloadData("")).
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client)
 
-	health, err := getWorkloadHealth(context.Background(), suite.runtime, workload.Id)
+		health, err := getWorkloadHealth(context.Background(), suite.runtime, workload.Id)
-	require.NoError(suite.T(), err)
+		require.NoError(suite.T(), err)
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_PASSING, health)
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_PASSING, health)
+	})
 }
 
 func (suite *getWorkloadHealthTestSuite) TestWithStatuses() {
@@ -626,24 +676,27 @@ func (suite *getWorkloadHealthTestSuite) TestWithStatuses() {
 	// helper method is used to inject multiple statuses in a way such that
 	// the resource service will return them in a predictable order and can
 	// properly exercise the code.
-	for value, status := range pbcatalog.Health_name {
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		health := pbcatalog.Health(value)
+		for value, status := range pbcatalog.Health_name {
-		if health == pbcatalog.Health_HEALTH_ANY {
+			health := pbcatalog.Health(value)
-			continue
+			if health == pbcatalog.Health_HEALTH_ANY {
+				continue
+			}
+
+			suite.Run(status, func() {
+				workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
+					WithData(suite.T(), workloadData("")).
+					WithTenancy(tenancy).
+					Write(suite.T(), suite.client)
+
+				suite.addHealthStatuses(workload.Id, tenancy, health)
+
+				actualHealth, err := getWorkloadHealth(context.Background(), suite.runtime, workload.Id)
+				require.NoError(suite.T(), err)
+				require.Equal(suite.T(), health, actualHealth)
+			})
 		}
-
+	})
-		suite.Run(status, func() {
-			workload := resourcetest.Resource(pbcatalog.WorkloadType, "foo").
-				WithData(suite.T(), workloadData("")).
-				Write(suite.T(), suite.client)
-
-			suite.addHealthStatuses(workload.Id, health)
-
-			actualHealth, err := getWorkloadHealth(context.Background(), suite.runtime, workload.Id)
-			require.NoError(suite.T(), err)
-			require.Equal(suite.T(), health, actualHealth)
-		})
-	}
 }
 
 func TestGetWorkloadHealth(t *testing.T) {
@@ -659,53 +712,62 @@ func (suite *getNodeHealthTestSuite) TestNotfound() {
 	// present in the system results in a the critical health but no error. This situation
 	// could occur when a linked node gets removed without the workloads being modified/removed.
 	// When that occurs we want to steer traffic away from the linked node as soon as possible.
-	health, err := getNodeHealth(context.Background(), suite.runtime, resourceID(pbcatalog.NodeType, "not-found"))
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-	require.NoError(suite.T(), err)
+		health, err := getNodeHealth(context.Background(), suite.runtime, resourceID(pbcatalog.NodeType, "not-found", tenancy))
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.NoError(suite.T(), err)
-
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getNodeHealthTestSuite) TestReadError() {
 	// This test's goal is to ensure the getNodeHealth propagates unexpected errors from
 	// its resource read call back to the caller.
-	health, err := getNodeHealth(context.Background(), suite.runtime, resourceID(fakeType, "not-found"))
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-	require.Error(suite.T(), err)
+		health, err := getNodeHealth(context.Background(), suite.runtime, resourceID(fakeType, "not-found", tenancy))
-	require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.Equal(suite.T(), codes.InvalidArgument, status.Code(err))
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getNodeHealthTestSuite) TestUnreconciled() {
 	// This test's goal is to ensure that nodes with unreconciled health are deemed
 	// critical. Basically, the workload health controller should defer calculating
 	// the workload health until the associated nodes health is known.
-	node := resourcetest.Resource(pbcatalog.NodeType, "unreconciled").
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		WithData(suite.T(), nodeData).
+		node := resourcetest.Resource(pbcatalog.NodeType, "unreconciled").
-		Write(suite.T(), suite.client).
+			WithData(suite.T(), nodeData).
-		GetId()
+			WithTenancy(tenancy).
+			Write(suite.T(), suite.client).
+			GetId()
 
-	health, err := getNodeHealth(context.Background(), suite.runtime, node)
+		health, err := getNodeHealth(context.Background(), suite.runtime, node)
-	require.Error(suite.T(), err)
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), errNodeUnreconciled, err)
+		require.Equal(suite.T(), errNodeUnreconciled, err)
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getNodeHealthTestSuite) TestNoConditions() {
 	// This test's goal is to ensure that if a node's health status doesn't have
-	// the expected condition then its deemedd critical. This should never happen
+	// the expected condition then its deemed critical. This should never happen
 	// in the integrated system as the node health controller would have to be
 	// buggy to add an empty status. However it could also indicate some breaking
 	// change went in. Regardless, the code to handle this state is written
 	// and it will be tested here.
-	node := resourcetest.Resource(pbcatalog.NodeType, "no-conditions").
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		WithData(suite.T(), nodeData).
+		node := resourcetest.Resource(pbcatalog.NodeType, "no-conditions").
-		WithStatus(nodehealth.StatusKey, &pbresource.Status{}).
+			WithData(suite.T(), nodeData).
-		Write(suite.T(), suite.client).
+			WithTenancy(tenancy).
-		GetId()
+			WithStatus(nodehealth.StatusKey, &pbresource.Status{}).
+			Write(suite.T(), suite.client).
+			GetId()
 
-	health, err := getNodeHealth(context.Background(), suite.runtime, node)
+		health, err := getNodeHealth(context.Background(), suite.runtime, node)
-	require.Error(suite.T(), err)
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), errNodeHealthConditionNotFound, err)
+		require.Equal(suite.T(), errNodeHealthConditionNotFound, err)
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getNodeHealthTestSuite) TestInvalidReason() {
@@ -716,48 +778,65 @@ func (suite *getNodeHealthTestSuite) TestInvalidReason() {
 	// controller to put it into this state. As users or other controllers could
 	// potentially force it into this state by writing the status themselves, it
 	// would be good to ensure the defined behavior works as expected.
-	node := resourcetest.Resource(pbcatalog.NodeType, "invalid-reason").
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		WithData(suite.T(), nodeData).
+		node := resourcetest.Resource(pbcatalog.NodeType, "invalid-reason").
-		WithStatus(nodehealth.StatusKey, &pbresource.Status{
+			WithData(suite.T(), nodeData).
-			Conditions: []*pbresource.Condition{
+			WithTenancy(tenancy).
-				{
+			WithStatus(nodehealth.StatusKey, &pbresource.Status{
-					Type:   nodehealth.StatusConditionHealthy,
+				Conditions: []*pbresource.Condition{
-					State:  pbresource.Condition_STATE_FALSE,
+					{
-					Reason: "INVALID_REASON",
+						Type:   nodehealth.StatusConditionHealthy,
+						State:  pbresource.Condition_STATE_FALSE,
+						Reason: "INVALID_REASON",
+					},
 				},
-			},
+			}).
-		}).
+			Write(suite.T(), suite.client).
-		Write(suite.T(), suite.client).
+			GetId()
-		GetId()
 
-	health, err := getNodeHealth(context.Background(), suite.runtime, node)
+		health, err := getNodeHealth(context.Background(), suite.runtime, node)
-	require.Error(suite.T(), err)
+		require.Error(suite.T(), err)
-	require.Equal(suite.T(), errNodeHealthInvalid, err)
+		require.Equal(suite.T(), errNodeHealthInvalid, err)
-	require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+		require.Equal(suite.T(), pbcatalog.Health_HEALTH_CRITICAL, health)
+	})
 }
 
 func (suite *getNodeHealthTestSuite) TestValidHealth() {
 	// This test aims to ensure that all status that would be reported by the node-health
 	// controller gets accurately detected and returned by the getNodeHealth function.
-	for value, healthStr := range pbcatalog.Health_name {
+	suite.controllerSuite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) {
-		health := pbcatalog.Health(value)
+		for value, healthStr := range pbcatalog.Health_name {
+			health := pbcatalog.Health(value)
 
-		// this is not a valid health that a health status
+			// this is not a valid health that a health status
-		// may be in.
+			// may be in.
-		if health == pbcatalog.Health_HEALTH_ANY {
+			if health == pbcatalog.Health_HEALTH_ANY {
-			continue
+				continue
+			}
+
+			suite.T().Run(healthStr, func(t *testing.T) {
+				node := suite.injectNodeWithStatus("test-node", health, tenancy)
+
+				actualHealth, err := getNodeHealth(context.Background(), suite.runtime, node.Id)
+				require.NoError(t, err)
+				require.Equal(t, health, actualHealth)
+			})
 		}
-
+	})
-		suite.T().Run(healthStr, func(t *testing.T) {
-			node := suite.injectNodeWithStatus("test-node", health)
-
-			actualHealth, err := getNodeHealth(context.Background(), suite.runtime, node.Id)
-			require.NoError(t, err)
-			require.Equal(t, health, actualHealth)
-		})
-	}
 }
 
 func TestGetNodeHealth(t *testing.T) {
 	suite.Run(t, new(getNodeHealthTestSuite))
 }
+
+func (suite *controllerSuite) runTestCaseWithTenancies(testFunc func(*pbresource.Tenancy)) {
+	for _, tenancy := range suite.tenancies {
+		suite.Run(suite.appendTenancyInfo(tenancy), func() {
+			testFunc(tenancy)
+		})
+	}
+}
+
+func (suite *controllerSuite) appendTenancyInfo(tenancy *pbresource.Tenancy) string {
+	return fmt.Sprintf("%s_Namespace_%s_Partition", tenancy.Namespace, tenancy.Partition)
+}

internal/resource/resourcetest/tenancy.go

@@ -7,10 +7,24 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/internal/resource"
 	"github.com/hashicorp/consul/proto-public/pbresource"
 )
 
+// TestTenancies returns a list of tenancies which represent
+// the namespace and partition combinations that can be used in unit tests
+func TestTenancies() []*pbresource.Tenancy {
+	isEnterprise := (structs.NodeEnterpriseMetaInDefaultPartition().PartitionOrEmpty() == "default")
+
+	tenancies := []*pbresource.Tenancy{Tenancy("default.default")}
+	if isEnterprise {
+		tenancies = append(tenancies, Tenancy("default.bar"), Tenancy("foo.default"), Tenancy("foo.bar"))
+	}
+
+	return tenancies
+}
+
 // Tenancy constructs a pbresource.Tenancy from a concise string representation
 // suitable for use in unit tests.
 //