From 51a18c2557560c3220872123dba03bda5838425e Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:08 +0200 Subject: [PATCH 1/6] docker: ignore "connection reset by peer" The Docker agent closes the connection during read after we have read the body. This causes a "connection reset by peer" even though the command was successful. We ignore that error here since we got the correct status code and a response body. --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 28c42a7571..46ff4adf03 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -155,7 +155,7 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { - case err != nil: + case err != nil && !strings.Contains(err.Error(), "connection reset by peer"): return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err) case code == 200: return b, nil From a1b47d99c1c038b59a5a64f903d8474c07fe3623 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:36 +0200 Subject: [PATCH 2/6] docker: make sure to log the error when we fall through --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 46ff4adf03..809a07252f 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -164,7 +164,7 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e case code == 409: return nil, fmt.Errorf("start exec failed since container %s is paused or stopped", containerID) default: - return nil, fmt.Errorf("start exec failed for container %s with status %d: %s", containerID, code, b) + return nil, fmt.Errorf("start exec failed for container %s with status %d: body: %s err: %s", containerID, code, b, err) } } From 94726ef105fc7feebced2f5e9fc0535046a05073 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:54 +0200 Subject: [PATCH 3/6] docker: do not alloc a tty since this is not interactive --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 809a07252f..c3cddd646b 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -151,7 +151,7 @@ func (c *DockerClient) CreateExec(containerID string, cmd []string) (string, err } func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, error) { - data := struct{ Detach, Tty bool }{Detach: false, Tty: true} + data := struct{ Detach, Tty bool }{Detach: false, Tty: false} uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { From e774b46f822fefc487b946a5baf000c210da42ab Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:57:18 +0200 Subject: [PATCH 4/6] docker: close idle connections on stop --- agent/checks/check.go | 1 + agent/checks/docker.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/agent/checks/check.go b/agent/checks/check.go index 446521021a..87e2c904b8 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -573,6 +573,7 @@ func (c *CheckDocker) Stop() { } func (c *CheckDocker) run() { + defer c.Client.Close() firstWait := lib.RandomStagger(c.Interval) next := time.After(firstWait) for { diff --git a/agent/checks/docker.go b/agent/checks/docker.go index c3cddd646b..c703cb0cb5 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -54,6 +54,13 @@ func NewDockerClient(host string, maxbuf int64) (*DockerClient, error) { }, nil } +func (c *DockerClient) Close() error { + if t, ok := c.client.Transport.(*http.Transport); ok { + t.CloseIdleConnections() + } + return nil +} + func (c *DockerClient) Host() string { return c.host } From 164ec3ec39ae73b1d762da79b47741a4a11e3974 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 12:03:07 +0200 Subject: [PATCH 5/6] docker: stop previous check on replace --- agent/agent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index 54628bca13..0dadd7025f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1776,6 +1776,9 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *structs.CheckType, Logger: a.logger, Client: a.dockerClient, } + if prev := a.checkDockers[check.CheckID]; prev != nil { + prev.Stop() + } dockerCheck.Start() a.checkDockers[check.CheckID] = dockerCheck From b4d8c26194eb7d51d01cc9a9f604919f6ee36357 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 12:14:19 +0200 Subject: [PATCH 6/6] docker: add comment about "connection reset by peer" error --- agent/checks/docker.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index c703cb0cb5..3644a68525 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -162,6 +162,12 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { + // todo(fs): https://github.com/hashicorp/consul/pull/3621 + // todo(fs): for some reason the docker agent closes the connection during the + // todo(fs): io.Copy call in c.call which causes a "connection reset by peer" error + // todo(fs): even though both body and status code have been received. My current is + // todo(fs): that the docker agent closes this prematurely but I don't understand why. + // todo(fs): the code below ignores this error. case err != nil && !strings.Contains(err.Error(), "connection reset by peer"): return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err) case code == 200: