diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 05e6b51045..68b10bbaa5 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -16,7 +16,7 @@ type Agent interface { GetClient() *api.Client GetName() string GetPod() testcontainers.Container - ClaimAdminPort() int + ClaimAdminPort() (int, error) GetConfig() Config GetInfo() AgentInfo GetDatacenter() string diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 0e339740e9..85a20d3c6d 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -23,6 +23,10 @@ import ( const bootLogLine = "Consul agent running" const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED" +// Exposed ports info +const MaxEnvoyOnNode = 10 // the max number of Envoy sidecar can run along with the agent, base is 19000 +const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream + // consulContainerNode implements the Agent interface by running a Consul agent // in a container. type consulContainerNode struct { @@ -55,10 +59,14 @@ func (c *consulContainerNode) GetPod() testcontainers.Container { return c.pod } -func (c *consulContainerNode) ClaimAdminPort() int { +func (c *consulContainerNode) ClaimAdminPort() (int, error) { + if c.nextAdminPortOffset >= MaxEnvoyOnNode { + return 0, fmt.Errorf("running out of envoy admin port, max %d, already claimed %d", + MaxEnvoyOnNode, c.nextAdminPortOffset) + } p := 19000 + c.nextAdminPortOffset c.nextAdminPortOffset++ - return p + return p, nil } // NewConsulContainer starts a Consul agent in a container with the given config. @@ -425,32 +433,29 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR Name: opts.name + "-pod", SkipReaper: skipReaper, ExposedPorts: []string{ - "8500/tcp", - "8501/tcp", + "8500/tcp", // Consul HTTP API + "8501/tcp", // Consul HTTPs API "8443/tcp", // Envoy Gateway Listener - "5000/tcp", // Envoy App Listener "8079/tcp", // Envoy App Listener "8080/tcp", // Envoy App Listener "9998/tcp", // Envoy App Listener "9999/tcp", // Envoy App Listener - - "19000/tcp", // Envoy Admin Port - "19001/tcp", // Envoy Admin Port - "19002/tcp", // Envoy Admin Port - "19003/tcp", // Envoy Admin Port - "19004/tcp", // Envoy Admin Port - "19005/tcp", // Envoy Admin Port - "19006/tcp", // Envoy Admin Port - "19007/tcp", // Envoy Admin Port - "19008/tcp", // Envoy Admin Port - "19009/tcp", // Envoy Admin Port }, Hostname: opts.hostname, Networks: opts.addtionalNetworks, } + // Envoy upstream listener + pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", ServiceUpstreamLocalBindPort)) + + // Reserve the exposed ports for Envoy admin port, e.g., 19000 - 19009 + basePort := 19000 + for i := 0; i < MaxEnvoyOnNode; i++ { + pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", basePort+i)) + } + // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". // If we only wait until the log stream starts, subsequent commands to agents will fail. // TODO: optimize the wait strategy diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 74cf21b213..80e9498bbe 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -107,7 +107,10 @@ func NewConnectService(ctx context.Context, name string, serviceName string, ser } dockerfileCtx.BuildArgs = buildargs - adminPort := node.ClaimAdminPort() + adminPort, err := node.ClaimAdminPort() + if err != nil { + return nil, err + } req := testcontainers.ContainerRequest{ FromDockerfile: dockerfileCtx, diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 2422aaafc4..90a79fe1a7 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -101,7 +101,10 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl } dockerfileCtx.BuildArgs = buildargs - adminPort := node.ClaimAdminPort() + adminPort, err := node.ClaimAdminPort() + if err != nil { + return nil, err + } req := testcontainers.ContainerRequest{ FromDockerfile: dockerfileCtx, diff --git a/test/integration/consul-container/libs/service/helpers.go b/test/integration/consul-container/libs/service/helpers.go index b4a2ad228c..21c06a3a21 100644 --- a/test/integration/consul-container/libs/service/helpers.go +++ b/test/integration/consul-container/libs/service/helpers.go @@ -96,7 +96,7 @@ func CreateAndRegisterStaticClientSidecar( DestinationName: StaticServerServiceName, DestinationPeer: peerName, LocalBindAddress: "0.0.0.0", - LocalBindPort: 5000, + LocalBindPort: libcluster.ServiceUpstreamLocalBindPort, MeshGateway: api.MeshGatewayConfig{ Mode: mgwMode, }, @@ -111,7 +111,7 @@ func CreateAndRegisterStaticClientSidecar( } // Create a service and proxy instance - clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, 5000, node) + clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, libcluster.ServiceUpstreamLocalBindPort, node) if err != nil { return nil, err }