ENT port: test-integ/peering: peer through mesh gateway [NET-4609] (#18605)

Co-authored-by: Matt Keeler <mjkeeler7@gmail.com>

test-integ/peering_commontopo/ac7_2_rotate_leader_test.go

@@ -196,7 +196,8 @@ func (s *ac7_2RotateLeaderSuite) test(t *testing.T, ct *commonTopo) {
 func rotateLeader(t *testing.T, cl *api.Client) {
 	t.Helper()
 	oldLeader := findLeader(t, cl)
-	cl.Operator().RaftLeaderTransfer(nil)
+	_, err := cl.Operator().RaftLeaderTransfer(nil)
+	require.NoError(t, err)
 	retry.RunWith(&retry.Timer{Timeout: 30 * time.Second, Wait: time.Second}, t, func(r *retry.R) {
 		newLeader := findLeader(r, cl)
 		require.NotEqual(r, oldLeader.ID, newLeader.ID)

test-integ/peering_commontopo/commontopo.go

@@ -59,12 +59,14 @@ func NewCommonTopo(t *testing.T) *commonTopo {
 
 	ct := commonTopo{}
 
+	const nServers = 3
+
 	// Make 3-server clusters in dc1 and dc2
 	// For simplicity, the Name and Datacenter of the clusters are the same.
 	// dc1 and dc2 should be symmetric.
-	dc1 := clusterWithJustServers("dc1", 3)
+	dc1 := clusterWithJustServers("dc1", nServers)
 	ct.DC1 = dc1
-	dc2 := clusterWithJustServers("dc2", 3)
+	dc2 := clusterWithJustServers("dc2", nServers)
 	ct.DC2 = dc2
 	// dc3 is a failover cluster for both dc1 and dc2
 	dc3 := clusterWithJustServers("dc3", 1)
@@ -367,6 +369,11 @@ func setupGlobals(clu *topology.Cluster) {
 					Mode: api.MeshGatewayModeLocal,
 				},
 			},
+			&api.MeshConfigEntry{
+				Peering: &api.PeeringMeshConfig{
+					PeerThroughMeshGateways: true,
+				},
+			},
 		)
 	}
 }
@@ -398,7 +405,7 @@ func clusterWithJustServers(name string, numServers int) *topology.Cluster {
 		Nodes: newTopologyServerSet(
 			name+"-server",
 			numServers,
-			[]string{name, "wan"},
+			[]string{name},
 			nil,
 		),
 	}

testing/deployer/sprawl/acl_rules.go

@@ -107,20 +107,26 @@ func tokenForService(svc *topology.Service, overridePolicy *api.ACLPolicy, enter
 	return token
 }
 
-func policyForMeshGateway(svc *topology.Service, enterprise bool) *api.ACLPolicy {
-	policyName := "mesh-gateway--" + svc.ID.ACLString()
-
-	policy := &api.ACLPolicy{
-		Name:        policyName,
-		Description: policyName,
+const (
+	meshGatewayCommunityRules = `
+service "mesh-gateway" {
+  policy = "write"
 }
-	if enterprise {
-		policy.Partition = svc.ID.Partition
-		policy.Namespace = "default"
+service_prefix "" {
+  policy = "read"
 }
+node_prefix "" {
+  policy = "read"
+}
+agent_prefix "" {
+  policy = "read"
+}
+# for peering
+mesh = "write"
+peering = "read"
+`
 
-	if enterprise {
-		policy.Rules = `
+	meshGatewayEntDefaultRules = `
 namespace_prefix "" {
   service "mesh-gateway" {
     policy = "write"
@@ -137,10 +143,14 @@ agent_prefix "" {
 }
 # for peering
 mesh = "write"
+
+partition_prefix "" {
   peering = "read"
+}
 `
-	} else {
-		policy.Rules = `
+
+	meshGatewayEntNonDefaultRules = `
+namespace_prefix "" {
   service "mesh-gateway" {
     policy = "write"
   }
@@ -150,13 +160,36 @@ service_prefix "" {
   node_prefix "" {
     policy = "read"
   }
+}
 agent_prefix "" {
   policy = "read"
 }
 # for peering
 mesh = "write"
-peering = "read"
 `
+)
+
+func policyForMeshGateway(svc *topology.Service, enterprise bool) *api.ACLPolicy {
+	policyName := "mesh-gateway--" + svc.ID.ACLString()
+
+	policy := &api.ACLPolicy{
+		Name:        policyName,
+		Description: policyName,
+	}
+	if enterprise {
+		fmt.Printf("Enterprise mgw ACLS - Partition: %s, Namespace: default", svc.ID.Partition)
+		policy.Partition = svc.ID.Partition
+		policy.Namespace = "default"
+	}
+
+	if enterprise {
+		if svc.ID.Partition == "default" {
+			policy.Rules = meshGatewayEntDefaultRules
+		} else {
+			policy.Rules = meshGatewayEntNonDefaultRules
+		}
+	} else {
+		policy.Rules = meshGatewayCommunityRules
 	}
 
 	return policy

testing/deployer/sprawl/peering.go

@@ -5,7 +5,9 @@ package sprawl
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 	"time"
 
@@ -89,7 +91,14 @@ func (s *Sprawl) initPeerings() error {
 				time.Sleep(50 * time.Millisecond)
 				goto ESTABLISH
 			}
-			return fmt.Errorf("error establishing peering with token for %q: %w", peering.String(), err)
+			// Establish and friends return an api.StatusError value, not pointer
+			// not sure if this is weird
+			var asStatusError api.StatusError
+			if errors.As(err, &asStatusError) && asStatusError.Code == http.StatusGatewayTimeout {
+				time.Sleep(50 * time.Millisecond)
+				goto ESTABLISH
+			}
+			return fmt.Errorf("error establishing peering with token for %q: %#v", peering.String(), err)
 		}
 
 		logger.Info("peering established", "peering", peering.String())