consul/agent/proxyprocess/daemon.go

package proxyprocess

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"reflect"
	"strconv"
	"sync"
	"time"

	"github.com/hashicorp/consul/lib/file"
	"github.com/mitchellh/mapstructure"
)

// Constants related to restart timers with the daemon mode proxies. At some
// point we will probably want to expose these knobs to an end user, but
// reasonable defaults are chosen.
const (
	DaemonRestartHealthy    = 10 * time.Second // time before considering healthy
	DaemonRestartBackoffMin = 3                // 3 attempts before backing off
	DaemonRestartMaxWait    = 1 * time.Minute  // maximum backoff wait time
)

// Daemon is a long-running proxy process. It is expected to keep running
// and to use blocking queries to detect changes in configuration, certs,
// and more.
//
// Consul will ensure that if the daemon crashes, that it is restarted.
type Daemon struct {
	// Command is the command to execute to start this daemon. This must
	// be a Cmd that isn't yet started.
	Command *exec.Cmd

	// ProxyID is the ID of the proxy service. This is required for API
	// requests (along with the token) and is passed via env var.
	ProxyID string

	// ProxyToken is the special local-only ACL token that allows a proxy
	// to communicate to the Connect-specific endpoints.
	ProxyToken string

	// Logger is where logs will be sent around the management of this
	// daemon. The actual logs for the daemon itself will be sent to
	// a file.
	Logger *log.Logger

	// PidPath is the path where a pid file will be created storing the
	// pid of the active process. If this is empty then a pid-file won't
	// be created. Under erroneous conditions, the pid file may not be
	// created but the error will be logged to the Logger.
	PidPath string

	// For tests, they can set this to change the default duration to wait
	// for a graceful quit.
	gracefulWait time.Duration

	// process is the started process
	lock     sync.Mutex
	stopped  bool
	stopCh   chan struct{}
	exitedCh chan struct{}
	process  *os.Process
}

// Start starts the daemon and keeps it running.
//
// This function returns after the process is successfully started.
func (p *Daemon) Start() error {
	p.lock.Lock()
	defer p.lock.Unlock()

	// A stopped proxy cannot be restarted
	if p.stopped {
		return fmt.Errorf("stopped")
	}

	// If we're already running, that is okay
	if p.process != nil {
		return nil
	}

	// Setup our stop channel
	stopCh := make(chan struct{})
	exitedCh := make(chan struct{})
	p.stopCh = stopCh
	p.exitedCh = exitedCh

	// Start the loop.
	go p.keepAlive(stopCh, exitedCh)

	return nil
}

// keepAlive starts and keeps the configured process alive until it
// is stopped via Stop.
func (p *Daemon) keepAlive(stopCh <-chan struct{}, exitedCh chan<- struct{}) {
	defer close(exitedCh)

	p.lock.Lock()
	process := p.process
	p.lock.Unlock()

	// attemptsDeadline is the time at which we consider the daemon to have
	// been alive long enough that we can reset the attempt counter.
	//
	// attempts keeps track of the number of restart attempts we've had and
	// is used to calculate the wait time using an exponential backoff.
	var attemptsDeadline time.Time
	var attempts uint32

	// Assume the process is adopted, we reset this when we start a new process
	// ourselves below and use it to decide on a strategy for waiting.
	adopted := true

	for {
		if process == nil {
			// If we're passed the attempt deadline then reset the attempts
			if !attemptsDeadline.IsZero() && time.Now().After(attemptsDeadline) {
				attempts = 0
			}
			// Set ourselves a deadline - we have to make it at least this long before
			// we come around the loop to consider it to have been a "successful"
			// daemon startup and rest the counter above. Note that if the daemon
			// fails before this, we reset the deadline to zero below so that backoff
			// sleeps in the loop don't count as "success" time.
			attemptsDeadline = time.Now().Add(DaemonRestartHealthy)
			attempts++

			// Calculate the exponential backoff and wait if we have to
			if attempts > DaemonRestartBackoffMin {
				exponent := (attempts - DaemonRestartBackoffMin)
				if exponent > 31 {
					exponent = 31
				}
				waitTime := (1 << exponent) * time.Second
				if waitTime > DaemonRestartMaxWait {
					waitTime = DaemonRestartMaxWait
				}

				if waitTime > 0 {
					// If we are waiting, reset the success deadline so we don't
					// accidentally interpret backoff sleep as successful runtime.
					attemptsDeadline = time.Time{}

					p.Logger.Printf(
						"[WARN] agent/proxy: waiting %s before restarting daemon",
						waitTime)

					timer := time.NewTimer(waitTime)
					select {
					case <-timer.C:
						// Timer is up, good!

					case <-stopCh:
						// During our backoff wait, we've been signalled to
						// quit, so just quit.
						timer.Stop()
						return
					}
				}
			}

			p.lock.Lock()

			// If we gracefully stopped then don't restart.
			if p.stopped {
				p.lock.Unlock()
				return
			}

			// Process isn't started currently. We're restarting. Start it
			// and save the process if we have it.
			var err error
			process, err = p.start()
			if err == nil {
				p.process = process
				adopted = false
			}
			p.lock.Unlock()

			if err != nil {
				p.Logger.Printf("[ERR] agent/proxy: error restarting daemon: %s", err)
				continue
			}

		}

		var ps *os.ProcessState
		var err error

		if adopted {
			// assign to err outside scope
			_, err = findProcess(process.Pid)
			if err == nil {
				// Process appears to be running still, wait a bit before we poll again.
				// We want a busy loop, but not too busy. 1 second between detecting a
				// process death seems reasonable.
				//
				// SUBTELTY: we must NOT select on stopCh here since the Stop function
				// assumes that as soon as this method returns and closes exitedCh, that
				// the process is no longer running. If we are polling then we don't
				// know that is true until we've polled again so we have to keep polling
				// until the process goes away even if we know the Daemon is stopping.
				time.Sleep(1 * time.Second)

				// Restart the loop, process is still set so we effectively jump back to
				// the findProcess call above.
				continue
			}
		} else {
			// Wait for child to exit
			ps, err = process.Wait()
		}

		// Process exited somehow.
		process = nil
		if err != nil {
			p.Logger.Printf("[INFO] agent/proxy: daemon exited with error: %s", err)
		} else if ps != nil && !ps.Exited() {
			p.Logger.Printf("[INFO] agent/proxy: daemon left running")
		} else if status, ok := exitStatus(ps); ok {
			p.Logger.Printf("[INFO] agent/proxy: daemon exited with exit code: %d", status)
		}
	}
}

// start starts and returns the process. This will create a copy of the
// configured *exec.Command with the modifications documented on Daemon
// such as setting the proxy token environmental variable.
func (p *Daemon) start() (*os.Process, error) {
	cmd := *p.Command

	// Add the proxy token to the environment. We first copy the env because it is
	// a slice and therefore the "copy" above will only copy the slice reference.
	// We allocate an exactly sized slice.
	//
	// Note that anything we add to the Env here is NOT persisted in the snapshot
	// which only looks at p.Command.Env so it needs to be reconstructible exactly
	// from data in the snapshot otherwise.
	cmd.Env = make([]string, len(p.Command.Env), len(p.Command.Env)+2)
	copy(cmd.Env, p.Command.Env)
	cmd.Env = append(cmd.Env,
		fmt.Sprintf("%s=%s", EnvProxyID, p.ProxyID),
		fmt.Sprintf("%s=%s", EnvProxyToken, p.ProxyToken))

	// Update the Daemon env

	// Args must always contain a 0 entry which is usually the executed binary.
	// To be safe and a bit more robust we default this, but only to prevent
	// a panic below.
	if len(cmd.Args) == 0 {
		cmd.Args = []string{cmd.Path}
	}

	// Perform system-specific setup. In particular, Unix-like systems
	// shuld set sid so that killing the agent doesn't kill the daemon.
	configureDaemon(&cmd)

	// Start it
	p.Logger.Printf("[DEBUG] agent/proxy: starting proxy: %q %#v", cmd.Path, cmd.Args[1:])
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	// Write the pid file. This might error and that's okay.
	if p.PidPath != "" {
		pid := strconv.FormatInt(int64(cmd.Process.Pid), 10)
		if err := file.WriteAtomic(p.PidPath, []byte(pid)); err != nil {
			p.Logger.Printf(
				"[DEBUG] agent/proxy: error writing pid file %q: %s",
				p.PidPath, err)
		}
	}

	return cmd.Process, nil
}

// Stop stops the daemon.
//
// This will attempt a graceful stop (SIGINT) before force killing the
// process (SIGKILL). In either case, the process won't be automatically
// restarted unless Start is called again.
//
// This is safe to call multiple times. If the daemon is already stopped,
// then this returns no error.
func (p *Daemon) Stop() error {
	p.lock.Lock()

	// If we're already stopped or never started, then no problem.
	if p.stopped || p.process == nil {
		// In the case we never even started, calling Stop makes it so
		// that we can't ever start in the future, either, so mark this.
		p.stopped = true
		p.lock.Unlock()
		return nil
	}

	// Note that we've stopped
	p.stopped = true
	close(p.stopCh)
	process := p.process
	p.lock.Unlock()

	gracefulWait := p.gracefulWait
	if gracefulWait == 0 {
		gracefulWait = 5 * time.Second
	}

	// Defer removing the pid file. Even under error conditions we
	// delete the pid file since Stop means that the manager is no
	// longer managing this proxy and therefore nothing else will ever
	// clean it up.
	if p.PidPath != "" {
		defer func() {
			if err := os.Remove(p.PidPath); err != nil && !os.IsNotExist(err) {
				p.Logger.Printf(
					"[DEBUG] agent/proxy: error removing pid file %q: %s",
					p.PidPath, err)
			}
		}()
	}

	// First, try a graceful stop
	err := process.Signal(os.Interrupt)
	if err == nil {
		select {
		case <-p.exitedCh:
			// Success!
			return nil

		case <-time.After(gracefulWait):
			// Interrupt didn't work
			p.Logger.Printf("[DEBUG] agent/proxy: graceful wait of %s passed, "+
				"killing", gracefulWait)
		}
	} else if isProcessAlreadyFinishedErr(err) {
		// This can happen due to races between signals and polling.
		return nil
	} else {
		p.Logger.Printf("[DEBUG] agent/proxy: sigint failed, killing: %s", err)
	}

	// Graceful didn't work (e.g. on windows where SIGINT isn't implemented),
	// forcibly kill
	err = process.Kill()
	if err != nil && isProcessAlreadyFinishedErr(err) {
		return nil
	}
	return err
}

// Close implements Proxy by stopping the run loop but not killing the process.
// One Close is called, Stop has no effect.
func (p *Daemon) Close() error {
	p.lock.Lock()
	defer p.lock.Unlock()

	// If we're already stopped or never started, then no problem.
	if p.stopped || p.process == nil {
		p.stopped = true
		return nil
	}

	// Note that we've stopped
	p.stopped = true
	close(p.stopCh)

	return nil
}

// Equal implements Proxy to check for equality.
func (p *Daemon) Equal(raw Proxy) bool {
	p2, ok := raw.(*Daemon)
	if !ok {
		return false
	}

	// We compare equality on a subset of the command configuration
	return p.ProxyToken == p2.ProxyToken &&
		p.ProxyID == p2.ProxyID &&
		p.Command.Path == p2.Command.Path &&
		p.Command.Dir == p2.Command.Dir &&
		reflect.DeepEqual(p.Command.Args, p2.Command.Args) &&
		reflect.DeepEqual(p.Command.Env, p2.Command.Env)
}

// MarshalSnapshot implements Proxy
func (p *Daemon) MarshalSnapshot() map[string]interface{} {
	p.lock.Lock()
	defer p.lock.Unlock()

	// If we're stopped or have no process, then nothing to snapshot.
	if p.stopped || p.process == nil {
		return nil
	}

	return map[string]interface{}{
		"Pid":         p.process.Pid,
		"CommandPath": p.Command.Path,
		"CommandArgs": p.Command.Args,
		"CommandDir":  p.Command.Dir,
		"CommandEnv":  p.Command.Env,
		"ProxyToken":  p.ProxyToken,
		"ProxyID":     p.ProxyID,
	}
}

// UnmarshalSnapshot implements Proxy
func (p *Daemon) UnmarshalSnapshot(m map[string]interface{}) error {
	var s daemonSnapshot
	if err := mapstructure.Decode(m, &s); err != nil {
		return err
	}

	p.lock.Lock()
	defer p.lock.Unlock()

	// Set the basic fields
	p.ProxyToken = s.ProxyToken
	p.ProxyID = s.ProxyID
	p.Command = &exec.Cmd{
		Path: s.CommandPath,
		Args: s.CommandArgs,
		Dir:  s.CommandDir,
		Env:  s.CommandEnv,
	}

	// FindProcess on many systems returns no error even if the process
	// is now dead. We perform an extra check that the process is alive.
	proc, err := findProcess(s.Pid)
	if err != nil {
		return err
	}

	// "Start it"
	stopCh := make(chan struct{})
	exitedCh := make(chan struct{})
	p.stopCh = stopCh
	p.exitedCh = exitedCh
	p.process = proc
	go p.keepAlive(stopCh, exitedCh)

	return nil
}

// daemonSnapshot is the structure of the marshalled data for snapshotting.
//
// Note we don't have to store the ProxyId because this is stored directly
// within the manager snapshot and is restored automatically.
type daemonSnapshot struct {
	// Pid of the process. This is the only value actually required to
	// regain management control. The remainder values are for Equal.
	Pid int

	// Command information
	CommandPath string
	CommandArgs []string
	CommandDir  string
	CommandEnv  []string

	// NOTE(mitchellh): longer term there are discussions/plans to only
	// store the hash of the token but for now we need the full token in
	// case the process dies and has to be restarted.
	ProxyToken string

	ProxyID string
}