From c26d5cf62cb943f74e08bcd95d3af4a54b255986 Mon Sep 17 00:00:00 2001 From: "R.B. Boyer" <4903+rboyer@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:39:09 -0500 Subject: [PATCH] test: fix container test enterprise drift (#19101) --- api/go.mod | 2 +- test/integration/consul-container/go.mod | 6 ++--- test/integration/consul-container/go.sum | 8 +++---- .../consul-container/libs/assert/envoy.go | 5 +++-- .../consul-container/libs/cluster/agent.go | 10 ++++----- .../consul-container/libs/cluster/cluster.go | 22 ++++++++++--------- .../libs/cluster/container.go | 19 ++++++++++++---- .../consul-container/libs/service/connect.go | 4 ++-- .../consul-container/libs/service/gateway.go | 5 +++-- .../consul-container/libs/service/helpers.go | 6 ++++- 10 files changed, 53 insertions(+), 34 deletions(-) diff --git a/api/go.mod b/api/go.mod index 0c6c4ee069..a9e1b8e353 100644 --- a/api/go.mod +++ b/api/go.mod @@ -9,6 +9,7 @@ require ( github.com/hashicorp/consul/sdk v0.14.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-hclog v1.5.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-rootcerts v1.0.2 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/serf v0.10.1 @@ -25,7 +26,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/go-version v1.2.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index 8917a5c5a2..3bca51fc21 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.2.1 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/serf v0.10.1 - github.com/itchyny/gojq v0.12.9 + github.com/itchyny/gojq v0.12.12 github.com/mitchellh/copystructure v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/otiai10/copy v1.10.0 @@ -31,6 +31,7 @@ require ( github.com/testcontainers/testcontainers-go v0.22.0 golang.org/x/mod v0.12.0 google.golang.org/grpc v1.57.0 + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 ) require ( @@ -138,7 +139,7 @@ require ( github.com/hashicorp/vault/sdk v0.7.0 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/imdario/mergo v0.3.15 // indirect - github.com/itchyny/timefmt-go v0.1.4 // indirect + github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -224,7 +225,6 @@ require ( k8s.io/client-go v0.26.2 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum index 2c33875443..eeb6f251dd 100644 --- a/test/integration/consul-container/go.sum +++ b/test/integration/consul-container/go.sum @@ -514,10 +514,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/itchyny/gojq v0.12.9 h1:biKpbKwMxVYhCU1d6mR7qMr3f0Hn9F5k5YykCVb3gmM= -github.com/itchyny/gojq v0.12.9/go.mod h1:T4Ip7AETUXeGpD+436m+UEl3m3tokRgajd5pRfsR5oE= -github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM= -github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA= +github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE= +github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= diff --git a/test/integration/consul-container/libs/assert/envoy.go b/test/integration/consul-container/libs/assert/envoy.go index 076367f1a6..760b1f87bc 100644 --- a/test/integration/consul-container/libs/assert/envoy.go +++ b/test/integration/consul-container/libs/assert/envoy.go @@ -14,12 +14,11 @@ import ( "testing" "time" + "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/go-cleanhttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/hashicorp/consul/sdk/testutil/retry" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -34,6 +33,7 @@ func GetEnvoyListenerTCPFilters(t *testing.T, adminPort int) { fmt.Sprintf("localhost:%d", adminPort), ) } + func GetEnvoyListenerTCPFiltersWithClient( t *testing.T, client *http.Client, @@ -85,6 +85,7 @@ func AssertUpstreamEndpointStatus(t *testing.T, adminPort int, clusterName, heal count, ) } + func AssertUpstreamEndpointStatusWithClient( t *testing.T, client *http.Client, diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index a86431a290..09568f21c5 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -10,15 +10,14 @@ import ( "io" jsonpatch "github.com/evanphx/json-patch" + agentconfig "github.com/hashicorp/consul/agent/config" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/lib/decode" "github.com/hashicorp/hcl" "github.com/mitchellh/mapstructure" "github.com/testcontainers/testcontainers-go" "google.golang.org/grpc" - agentconfig "github.com/hashicorp/consul/agent/config" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/lib/decode" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -79,7 +78,8 @@ type Config struct { UseAPIWithTLS bool // TODO UseGRPCWithTLS bool - ACLEnabled bool + ACLEnabled bool + TokenBootstrap string } func (c *Config) DockerImage() string { diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index 66e9ac5c2d..fbee48333e 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -15,15 +15,15 @@ import ( "testing" "time" + goretry "github.com/avast/retry-go" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil/retry" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "github.com/hashicorp/serf/serf" - - goretry "github.com/avast/retry-go" "github.com/stretchr/testify/require" "github.com/teris-io/shortid" "github.com/testcontainers/testcontainers-go" + + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) // Cluster provides an interface for creating and controlling a Consul cluster @@ -93,11 +93,12 @@ func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) { } cluster := &Cluster{ - ID: id, - Network: network, - NetworkName: name, - ScratchDir: scratchDir, - ACLEnabled: configs[0].ACLEnabled, + ID: id, + Network: network, + NetworkName: name, + ScratchDir: scratchDir, + ACLEnabled: configs[0].ACLEnabled, + TokenBootstrap: configs[0].TokenBootstrap, } t.Cleanup(func() { _ = cluster.Terminate() @@ -193,8 +194,8 @@ func (c *Cluster) join(agents []Agent, skipSerfJoin bool) error { } if len(c.Agents) == 0 { - // if acl enabled, generate the bootstrap tokens at the first agent - if c.ACLEnabled { + // if acl enabled and bootstrap token is null, generate the bootstrap tokens at the first agent + if c.ACLEnabled && c.TokenBootstrap == "" { var ( output string err error @@ -598,6 +599,7 @@ func (c *Cluster) PeerWithCluster(acceptingClient *api.Client, acceptingPeerName } const retryTimeout = 90 * time.Second + const retryFrequency = 500 * time.Millisecond func LongFailer() *retry.Timer { diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 481436f34c..0ff9038c0e 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -10,6 +10,7 @@ import ( "io" "net/url" "os" + "os/exec" "path/filepath" "strconv" "time" @@ -17,6 +18,7 @@ import ( goretry "github.com/avast/retry-go" dockercontainer "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" + "github.com/hashicorp/consul/api" "github.com/hashicorp/go-multierror" "github.com/otiai10/copy" "github.com/pkg/errors" @@ -25,12 +27,11 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) const bootLogLine = "Consul agent running" + const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED" // Exposed ports info @@ -135,6 +136,13 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po if err != nil { return nil, fmt.Errorf("error copying persistent data from %s: %w", config.ExternalDataDir, err) } + // NOTE: make sure the new version can access the persistent data + // This is necessary for running on Linux + cmd := exec.Command("chmod", "-R", "777", tmpDirData) + err = cmd.Run() + if err != nil { + return nil, fmt.Errorf("error changing ownership of persistent data: %w", err) + } } var caCertFileForAPI string @@ -337,8 +345,9 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po node.clientCACertFile = clientCACertFile } - // Inject node token if ACL is enabled and the bootstrap token is generated - if cluster.TokenBootstrap != "" && cluster.ACLEnabled { + // Inject node token if ACL is enabled, the bootstrap token not null, and cluster + // has at least one agent + if cluster.TokenBootstrap != "" && cluster.ACLEnabled && len(cluster.Agents) > 0 { agentToken, err := cluster.CreateAgentToken(pc.Datacenter, name) if err != nil { return nil, err @@ -534,9 +543,11 @@ func (c *consulContainerNode) Upgrade(ctx context.Context, config Config) error func (c *consulContainerNode) Terminate() error { return c.terminate(false, false) } + func (c *consulContainerNode) TerminateAndRetainPod(skipFuncs bool) error { return c.terminate(true, skipFuncs) } + func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error { // Services might register a termination function that should also fire // when the "agent" is cleaned up. diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 6d283eac67..4b9998a779 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -13,11 +13,10 @@ import ( "strings" "time" + "github.com/hashicorp/consul/api" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -206,6 +205,7 @@ func NewConnectService( "-sidecar-for", sidecarCfg.ServiceID, "-admin-bind", fmt.Sprintf("0.0.0.0:%d", internalAdminPort), "-namespace", sidecarCfg.Namespace, + "-partition", sidecarCfg.Partition, "--", "--log-level", envoyLogLevel, }, diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 61bc4d3150..b468d9ecf1 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -11,11 +11,10 @@ import ( "strconv" "time" + "github.com/hashicorp/consul/api" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" - "github.com/hashicorp/consul/api" - libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -145,6 +144,7 @@ type GatewayConfig struct { Name string Kind string Namespace string + Partition string } func NewGatewayService(ctx context.Context, gwCfg GatewayConfig, node libcluster.Agent, ports ...int) (Service, error) { @@ -170,6 +170,7 @@ func NewGatewayServiceReg(ctx context.Context, gwCfg GatewayConfig, node libclus fmt.Sprintf("-gateway=%s", gwCfg.Kind), "-service", gwCfg.Name, "-namespace", gwCfg.Namespace, + "-partition", gwCfg.Partition, "-address", "{{ GetInterfaceIP \"eth0\" }}:8443", "-admin-bind", fmt.Sprintf("0.0.0.0:%d", adminPort), } diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index a524d9c040..eb4cb2d19f 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -8,10 +8,10 @@ import ( "fmt" "testing" + "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" - "github.com/hashicorp/consul/api" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" ) @@ -49,6 +49,7 @@ type ServiceOpts struct { Namespace string Partition string Locality *api.Locality + Upstreams []api.Upstream } // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers @@ -322,6 +323,9 @@ func CreateAndRegisterStaticClientSidecar( if serviceOpts.Connect.Port != 0 { req.Connect.SidecarService.Port = serviceOpts.Connect.Port } + if len(serviceOpts.Upstreams) > 0 { + req.Connect.SidecarService.Proxy.Upstreams = serviceOpts.Upstreams + } req.Meta = serviceOpts.Meta req.Namespace = serviceOpts.Namespace req.Partition = serviceOpts.Partition