test: fix container test enterprise drift (#19101)

api/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/hashicorp/consul/sdk v0.14.1
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-hclog v1.5.0
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-rootcerts v1.0.2
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/serf v0.10.1
@@ -25,7 +26,6 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/go-version v1.2.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect

test/integration/consul-container/go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/hashicorp/go-version v1.2.1
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/serf v0.10.1
-	github.com/itchyny/gojq v0.12.9
+	github.com/itchyny/gojq v0.12.12
 	github.com/mitchellh/copystructure v1.2.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/otiai10/copy v1.10.0
@@ -31,6 +31,7 @@ require (
 	github.com/testcontainers/testcontainers-go v0.22.0
 	golang.org/x/mod v0.12.0
 	google.golang.org/grpc v1.57.0
+	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 )
 
 require (
@@ -138,7 +139,7 @@ require (
 	github.com/hashicorp/vault/sdk v0.7.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
-	github.com/itchyny/timefmt-go v0.1.4 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -224,7 +225,6 @@ require (
 	k8s.io/client-go v0.26.2 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

test/integration/consul-container/go.sum

@@ -514,10 +514,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/itchyny/gojq v0.12.9 h1:biKpbKwMxVYhCU1d6mR7qMr3f0Hn9F5k5YykCVb3gmM=
-github.com/itchyny/gojq v0.12.9/go.mod h1:T4Ip7AETUXeGpD+436m+UEl3m3tokRgajd5pRfsR5oE=
-github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM=
-github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
+github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA=
+github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE=
+github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
+github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=

test/integration/consul-container/libs/assert/envoy.go

@@ -14,12 +14,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/hashicorp/consul/sdk/testutil/retry"
-
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -34,6 +33,7 @@ func GetEnvoyListenerTCPFilters(t *testing.T, adminPort int) {
 		fmt.Sprintf("localhost:%d", adminPort),
 	)
 }
+
 func GetEnvoyListenerTCPFiltersWithClient(
 	t *testing.T,
 	client *http.Client,
@@ -85,6 +85,7 @@ func AssertUpstreamEndpointStatus(t *testing.T, adminPort int, clusterName, heal
 		count,
 	)
 }
+
 func AssertUpstreamEndpointStatusWithClient(
 	t *testing.T,
 	client *http.Client,

test/integration/consul-container/libs/cluster/agent.go

@@ -10,15 +10,14 @@ import (
 	"io"
 
 	jsonpatch "github.com/evanphx/json-patch"
+	agentconfig "github.com/hashicorp/consul/agent/config"
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/lib/decode"
 	"github.com/hashicorp/hcl"
 	"github.com/mitchellh/mapstructure"
 	"github.com/testcontainers/testcontainers-go"
 	"google.golang.org/grpc"
 
-	agentconfig "github.com/hashicorp/consul/agent/config"
-	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/lib/decode"
-
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
@@ -80,6 +79,7 @@ type Config struct {
 	UseGRPCWithTLS bool
 
 	ACLEnabled     bool
+	TokenBootstrap string
 }
 
 func (c *Config) DockerImage() string {

test/integration/consul-container/libs/cluster/cluster.go

@@ -15,15 +15,15 @@ import (
 	"testing"
 	"time"
 
+	goretry "github.com/avast/retry-go"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
-	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 	"github.com/hashicorp/serf/serf"
-
-	goretry "github.com/avast/retry-go"
 	"github.com/stretchr/testify/require"
 	"github.com/teris-io/shortid"
 	"github.com/testcontainers/testcontainers-go"
+
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
 // Cluster provides an interface for creating and controlling a Consul cluster
@@ -98,6 +98,7 @@ func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) {
 		NetworkName:    name,
 		ScratchDir:     scratchDir,
 		ACLEnabled:     configs[0].ACLEnabled,
+		TokenBootstrap: configs[0].TokenBootstrap,
 	}
 	t.Cleanup(func() {
 		_ = cluster.Terminate()
@@ -193,8 +194,8 @@ func (c *Cluster) join(agents []Agent, skipSerfJoin bool) error {
 	}
 
 	if len(c.Agents) == 0 {
-		// if acl enabled, generate the bootstrap tokens at the first agent
-		if c.ACLEnabled {
+		// if acl enabled and bootstrap token is null, generate the bootstrap tokens at the first agent
+		if c.ACLEnabled && c.TokenBootstrap == "" {
 			var (
 				output string
 				err    error
@@ -598,6 +599,7 @@ func (c *Cluster) PeerWithCluster(acceptingClient *api.Client, acceptingPeerName
 }
 
 const retryTimeout = 90 * time.Second
+
 const retryFrequency = 500 * time.Millisecond
 
 func LongFailer() *retry.Timer {

test/integration/consul-container/libs/cluster/container.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -17,6 +18,7 @@ import (
 	goretry "github.com/avast/retry-go"
 	dockercontainer "github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/go-multierror"
 	"github.com/otiai10/copy"
 	"github.com/pkg/errors"
@@ -25,12 +27,11 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
-	"github.com/hashicorp/consul/api"
-
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
 const bootLogLine = "Consul agent running"
+
 const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED"
 
 // Exposed ports info
@@ -135,6 +136,13 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		if err != nil {
 			return nil, fmt.Errorf("error copying persistent data from %s: %w", config.ExternalDataDir, err)
 		}
+		// NOTE: make sure the new version can access the persistent data
+		// This is necessary for running on Linux
+		cmd := exec.Command("chmod", "-R", "777", tmpDirData)
+		err = cmd.Run()
+		if err != nil {
+			return nil, fmt.Errorf("error changing ownership of persistent data: %w", err)
+		}
 	}
 
 	var caCertFileForAPI string
@@ -337,8 +345,9 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		node.clientCACertFile = clientCACertFile
 	}
 
-	// Inject node token if ACL is enabled and the bootstrap token is generated
-	if cluster.TokenBootstrap != "" && cluster.ACLEnabled {
+	// Inject node token if ACL is enabled, the bootstrap token not null, and cluster
+	// has at least one agent
+	if cluster.TokenBootstrap != "" && cluster.ACLEnabled && len(cluster.Agents) > 0 {
 		agentToken, err := cluster.CreateAgentToken(pc.Datacenter, name)
 		if err != nil {
 			return nil, err
@@ -534,9 +543,11 @@ func (c *consulContainerNode) Upgrade(ctx context.Context, config Config) error
 func (c *consulContainerNode) Terminate() error {
 	return c.terminate(false, false)
 }
+
 func (c *consulContainerNode) TerminateAndRetainPod(skipFuncs bool) error {
 	return c.terminate(true, skipFuncs)
 }
+
 func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error {
 	// Services might register a termination function that should also fire
 	// when the "agent" is cleaned up.

test/integration/consul-container/libs/service/connect.go

@@ -13,11 +13,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
 
-	"github.com/hashicorp/consul/api"
-
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -206,6 +205,7 @@ func NewConnectService(
 			"-sidecar-for", sidecarCfg.ServiceID,
 			"-admin-bind", fmt.Sprintf("0.0.0.0:%d", internalAdminPort),
 			"-namespace", sidecarCfg.Namespace,
+			"-partition", sidecarCfg.Partition,
 			"--",
 			"--log-level", envoyLogLevel,
 		},

test/integration/consul-container/libs/service/gateway.go

@@ -11,11 +11,10 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
 
-	"github.com/hashicorp/consul/api"
-
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -145,6 +144,7 @@ type GatewayConfig struct {
 	Name      string
 	Kind      string
 	Namespace string
+	Partition string
 }
 
 func NewGatewayService(ctx context.Context, gwCfg GatewayConfig, node libcluster.Agent, ports ...int) (Service, error) {
@@ -170,6 +170,7 @@ func NewGatewayServiceReg(ctx context.Context, gwCfg GatewayConfig, node libclus
 		fmt.Sprintf("-gateway=%s", gwCfg.Kind),
 		"-service", gwCfg.Name,
 		"-namespace", gwCfg.Namespace,
+		"-partition", gwCfg.Partition,
 		"-address", "{{ GetInterfaceIP \"eth0\" }}:8443",
 		"-admin-bind", fmt.Sprintf("0.0.0.0:%d", adminPort),
 	}

test/integration/consul-container/libs/service/helpers.go

@@ -8,10 +8,10 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 
-	"github.com/hashicorp/consul/api"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
@@ -49,6 +49,7 @@ type ServiceOpts struct {
 	Namespace    string
 	Partition    string
 	Locality     *api.Locality
+	Upstreams    []api.Upstream
 }
 
 // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers
@@ -322,6 +323,9 @@ func CreateAndRegisterStaticClientSidecar(
 		if serviceOpts.Connect.Port != 0 {
 			req.Connect.SidecarService.Port = serviceOpts.Connect.Port
 		}
+		if len(serviceOpts.Upstreams) > 0 {
+			req.Connect.SidecarService.Proxy.Upstreams = serviceOpts.Upstreams
+		}
 		req.Meta = serviceOpts.Meta
 		req.Namespace = serviceOpts.Namespace
 		req.Partition = serviceOpts.Partition