consul/command/connect/envoy/pipe-bootstrap/connect_envoy_pipe-bootstrap.go
Daniel Nephin c9bc5f92b7 envoy: fix bootstrap deadlock caused by a full named pipe
Normally the named pipe would buffer up to 64k, but in some cases when a
soft limit is reached, they will start only buffering up to 4k.
In either case, we should not deadlock.

This commit changes the pipe-bootstrap command to first buffer all of
stdin into the process, before trying to write it to the named pipe.
This allows the process memory to act as the buffer, instead of the
named pipe.

Also changed the order of operations in `makeBootstrapPipe`. The new
test added in this PR showed that simply buffering in the process memory
was not enough to fix the issue. We also need to ensure that the
`pipe-bootstrap` process is started before we try to write to its
stdin. Otherwise the write will still block.

Also set stdout/stderr on the subprocess, so that any errors are visible
to the user.
2021-05-31 18:53:17 -04:00

94 lines
2.0 KiB
Go

package pipebootstrap
import (
"bytes"
"flag"
"os"
"time"
"github.com/mitchellh/cli"
"github.com/hashicorp/consul/command/flags"
)
func New(ui cli.Ui) *cmd {
c := &cmd{UI: ui}
c.init()
return c
}
type cmd struct {
UI cli.Ui
flags *flag.FlagSet
help string
}
func (c *cmd) init() {
c.help = flags.Usage(help, c.flags)
}
func (c *cmd) Run(args []string) int {
// Read from STDIN, write to the named pipe provided in the only positional arg
if len(args) != 1 {
c.UI.Error("Expecting named pipe path as argument")
return 1
}
// This should never be alive for very long. In case bad things happen and
// Envoy never starts limit how long we live before just exiting so we can't
// accumulate tons of these zombie children.
time.AfterFunc(10*time.Second, func() {
// Force cleanup
os.RemoveAll(args[0])
os.Exit(99)
})
var buf bytes.Buffer
if _, err := buf.ReadFrom(os.Stdin); err != nil {
c.UI.Error(err.Error())
return 1
}
// WRONLY is very important here - the open() call will block until there is a
// reader (Envoy) if we open it with RDWR though that counts as an opener and
// we will just send the data to ourselves as the first opener and so only
// valid reader.
f, err := os.OpenFile(args[0], os.O_WRONLY|os.O_APPEND, 0700)
if err != nil {
c.UI.Error(err.Error())
return 1
}
if _, err := buf.WriteTo(f); err != nil {
c.UI.Error(err.Error())
return 1
}
if err = f.Close(); err != nil {
c.UI.Error(err.Error())
return 1
}
// Removed named pipe now we sent it. Even if Envoy has not yet read it, we
// know it has opened it and has the file descriptor since our write above
// will block until there is a reader.
c.UI.Warn("Bootstrap sent, unlinking named pipe")
os.RemoveAll(args[0])
return 0
}
func (c *cmd) Synopsis() string {
return synopsis
}
func (c *cmd) Help() string {
return c.help
}
const synopsis = "Internal shim for delivering Envoy bootstrap without writing to file system"
const help = `
Usage: should only be used internally by consul connect envoy
`