Use null route cluster for default router when no matches on v2 mesh gateway (#20270)

* Use black hole cluster for default router when no matches

* Update test assertions

* Use null route cluster instead of black hole cluster concept

* Update test assertions

internal/mesh/internal/controllers/gatewayproxy/builder/builder.go

@@ -6,8 +6,10 @@ package builder
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/hashicorp/go-hclog"
+	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
@@ -22,16 +24,18 @@ import (
 	"github.com/hashicorp/consul/proto-public/pbresource"
 )
 
+const nullRouteClusterName = "null_route_cluster"
+
 type proxyStateTemplateBuilder struct {
 	workload         *types.DecodedWorkload
 	dataFetcher      *fetcher.Fetcher
 	dc               string
-	exportedServices *types.DecodedComputedExportedServices
+	exportedServices []*pbmulticluster.ComputedExportedService
 	logger           hclog.Logger
 	trustDomain      string
 }
 
-func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload, exportedServices *types.DecodedComputedExportedServices, logger hclog.Logger, dataFetcher *fetcher.Fetcher, dc, trustDomain string) *proxyStateTemplateBuilder {
+func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload, exportedServices []*pbmulticluster.ComputedExportedService, logger hclog.Logger, dataFetcher *fetcher.Fetcher, dc, trustDomain string) *proxyStateTemplateBuilder {
 	return &proxyStateTemplateBuilder{
 		workload:         workload,
 		dataFetcher:      dataFetcher,
@@ -100,7 +104,7 @@ func (b *proxyStateTemplateBuilder) buildListener(address *pbcatalog.WorkloadAdd
 				L4: &pbproxystate.L4Destination{
 					Destination: &pbproxystate.L4Destination_Cluster{
 						Cluster: &pbproxystate.DestinationCluster{
-							Name: "",
+							Name: nullRouteClusterName,
 						},
 					},
 					StatPrefix: "prefix",
@@ -117,11 +121,7 @@ func (b *proxyStateTemplateBuilder) buildListener(address *pbcatalog.WorkloadAdd
 func (b *proxyStateTemplateBuilder) routers() []*pbproxystate.Router {
 	var routers []*pbproxystate.Router
 
-	if b.exportedServices == nil {
+	for _, exportedService := range b.exportedServices {
-		return routers
-	}
-
-	for _, exportedService := range b.exportedServices.Data.Services {
 		serviceID := resource.IDFromReference(exportedService.TargetRef)
 		service, err := b.dataFetcher.FetchService(context.Background(), serviceID)
 		if err != nil {
@@ -160,11 +160,7 @@ func (b *proxyStateTemplateBuilder) routers() []*pbproxystate.Router {
 func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster {
 	clusters := map[string]*pbproxystate.Cluster{}
 
-	if b.exportedServices == nil {
+	for _, exportedService := range b.exportedServices {
-		return clusters
-	}
-
-	for _, exportedService := range b.exportedServices.Data.Services {
 		serviceID := resource.IDFromReference(exportedService.TargetRef)
 		service, err := b.dataFetcher.FetchService(context.Background(), serviceID)
 		if err != nil {
@@ -192,6 +188,23 @@ func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster
 		}
 	}
 
+	// Add null route cluster for any unmatched traffic
+	clusters[nullRouteClusterName] = &pbproxystate.Cluster{
+		Name: nullRouteClusterName,
+		Group: &pbproxystate.Cluster_EndpointGroup{
+			EndpointGroup: &pbproxystate.EndpointGroup{
+				Group: &pbproxystate.EndpointGroup_Static{
+					Static: &pbproxystate.StaticEndpointGroup{
+						Config: &pbproxystate.StaticEndpointGroupConfig{
+							ConnectTimeout: durationpb.New(10 * time.Second),
+						},
+					},
+				},
+			},
+		},
+		Protocol: pbproxystate.Protocol_PROTOCOL_TCP,
+	}
+
 	return clusters
 }
 
@@ -219,11 +232,7 @@ func (b *proxyStateTemplateBuilder) Build() *meshv2beta1.ProxyStateTemplate {
 func (b *proxyStateTemplateBuilder) requiredEndpoints() map[string]*pbproxystate.EndpointRef {
 	requiredEndpoints := make(map[string]*pbproxystate.EndpointRef)
 
-	if b.exportedServices == nil {
+	for _, exportedService := range b.exportedServices {
-		return requiredEndpoints
-	}
-
-	for _, exportedService := range b.exportedServices.Data.Services {
 		serviceID := resource.IDFromReference(exportedService.TargetRef)
 		service, err := b.dataFetcher.FetchService(context.Background(), serviceID)
 		if err != nil {

internal/mesh/internal/controllers/gatewayproxy/builder/builder_test.go

@@ -7,9 +7,11 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/hashicorp/consul/agent/connect"
 	svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing"
@@ -197,7 +199,7 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build
 			"without address ports": suite.workloadWithOutAddressPorts,
 		} {
 			testutil.RunStep(suite.T(), name, func(t *testing.T) {
-				builder := NewProxyStateTemplateBuilder(workload, suite.exportedServicesPeerData, logger, f, dc, trustDomain)
+				builder := NewProxyStateTemplateBuilder(workload, suite.exportedServicesPeerData.Data.Services, logger, f, dc, trustDomain)
 				expectedProxyStateTemplate := &pbmesh.ProxyStateTemplate{
 					ProxyState: &pbmesh.ProxyState{
 						Identity: &pbresource.Reference{
@@ -223,7 +225,7 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build
 										L4: &pbproxystate.L4Destination{
 											Destination: &pbproxystate.L4Destination_Cluster{
 												Cluster: &pbproxystate.DestinationCluster{
-													Name: "",
+													Name: nullRouteClusterName,
 												},
 											},
 											StatPrefix: "prefix",
@@ -267,6 +269,21 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build
 							},
 						},
 						Clusters: map[string]*pbproxystate.Cluster{
+							nullRouteClusterName: {
+								Name: nullRouteClusterName,
+								Group: &pbproxystate.Cluster_EndpointGroup{
+									EndpointGroup: &pbproxystate.EndpointGroup{
+										Group: &pbproxystate.EndpointGroup_Static{
+											Static: &pbproxystate.StaticEndpointGroup{
+												Config: &pbproxystate.StaticEndpointGroupConfig{
+													ConnectTimeout: durationpb.New(10 * time.Second),
+												},
+											},
+										},
+									},
+								},
+								Protocol: pbproxystate.Protocol_PROTOCOL_TCP,
+							},
 							fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")): {
 								Name: fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")),
 								Group: &pbproxystate.Cluster_EndpointGroup{

internal/mesh/internal/controllers/gatewayproxy/controller.go

@@ -119,15 +119,14 @@ func (r *reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req c
 		Type: pbmulticluster.ComputedExportedServicesType,
 	}
 
-	exportedServices, err := dataFetcher.FetchExportedServices(ctx, exportedServicesID)
+	var exportedServices []*pbmulticluster.ComputedExportedService
-	if err != nil || exportedServices == nil {
+	dec, err := dataFetcher.FetchExportedServices(ctx, exportedServicesID)
-		if err != nil {
+	if err != nil {
-			rt.Logger.Error("error reading the associated exported services", "error", err)
+		rt.Logger.Error("error reading the associated exported services", "error", err)
-		}
+	} else if dec == nil {
-
+		rt.Logger.Error("exported services was nil")
-		if exportedServices == nil {
+	} else {
-			rt.Logger.Error("exported services was nil")
+		exportedServices = dec.Data.Services
-		}
 	}
 
 	trustDomain, err := r.getTrustDomain()