status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Debug failing CICD tests (#20455) 

Fix CICD test flakes by locking container socket.
This commit is contained in:
Derek Menteer 2024-02-02 15:05:10 -06:00 committed by GitHub
parent fb2b696c0e
commit 266f6548f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
test/integration/consul-container/libs/cluster/container.go
View File

@ -13,6 +13,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv" "strconv"
"sync"
"time" "time"
goretry "github.com/avast/retry-go" goretry "github.com/avast/retry-go"
@ -40,6 +41,16 @@ const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstr
const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams
const debugPort = "4000/tcp" const debugPort = "4000/tcp"
// containerLock prevents starting multiple containers concurrently. This has not been confirmed as being necessary, but
// it seems to help make the CICD pipeline pass without failures. These failures seem to be due to some form of docker
// socket contention with errors of the form:
//
// #1: error starting pod with image "docker.mirror.hashicorp.services/hashiderek/pause": Post "http://%2Fvar%2Frun%2Fdocker.sock/v1.43/containers/9b0e568744793e558d318af908c1052ab3d4d2f5a74c67b15d47a0570f141b1c/start": context deadline exceeded: failed to start container
//
// It may purely be due to the fact that starting containers takes longer than expected, and this lock avoids starting
// the context cancel timer until after we have ensured the docker socket is freed up.
var containerLock sync.Mutex
// consulContainerNode implements the Agent interface by running a Consul agent // consulContainerNode implements the Agent interface by running a Consul agent
// in a container. // in a container.
type consulContainerNode struct { type consulContainerNode struct {
@ -604,6 +615,8 @@ func (c *consulContainerNode) DataDir() string {
} }
func startContainer(ctx context.Context, req testcontainers.ContainerRequest) (testcontainers.Container, error) { func startContainer(ctx context.Context, req testcontainers.ContainerRequest) (testcontainers.Container, error) {
containerLock.Lock()
defer containerLock.Unlock()
ctx, cancel := context.WithTimeout(ctx, time.Second*40) ctx, cancel := context.WithTimeout(ctx, time.Second*40)
defer cancel() defer cancel()
return testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ return testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{