ACL Token Persistence and Reloading (#5328)

This PR adds two features which will be useful for operators when ACLs are in use.

1. Tokens set in configuration files are now reloadable.
2. If `acl.enable_token_persistence` is set to `true` in the configuration, tokens set via the `v1/agent/token` endpoint are now persisted to disk and loaded when the agent starts (or during configuration reload)

Note that token persistence is opt-in so our users who do not want tokens on the local disk will see no change.

Some other secondary changes:

* Refactored a bunch of places where the replication token is retrieved from the token store. This token isn't just for replicating ACLs and now it is named accordingly.
* Allowed better paths in the `v1/agent/token/` API. Instead of paths like: `v1/agent/token/acl_replication_token` the path can now be just `v1/agent/token/replication`. The old paths remain to be valid. 
* Added a couple new API functions to set tokens via the new paths. Deprecated the old ones and pointed to the new names. The names are also generally better and don't imply that what you are setting is for ACLs but rather are setting ACL tokens. There is a minor semantic difference there especially for the replication token as again, its no longer used only for ACL token/policy replication. The new functions will detect 404s and fallback to using the older token paths when talking to pre-1.4.3 agents.
* Docs updated to reflect the API additions and to show using the new endpoints.
* Updated the ACL CLI set-agent-tokens command to use the non-deprecated APIs.

agent/acl_test.go

@@ -190,6 +190,7 @@ func TestACL_AgentMasterToken(t *testing.T) {
 	}
 
 	a := NewTestACLAgent(t.Name(), TestACLConfig(), resolveFn)
+	a.loadTokens(a.config)
 	authz, err := a.resolveToken("towel")
 	require.NotNil(t, authz)
 	require.Nil(t, err)

agent/agent.go

@@ -63,6 +63,9 @@ const (
 	checksDir     = "checks"
 	checkStateDir = "checks/state"
 
+	// Name of the file tokens will be persisted within
+	tokensPath = "acl-tokens.json"
+
 	// Default reasons for node/service maintenance mode
 	defaultNodeMaintReason = "Maintenance mode is enabled for this node, " +
 		"but no reason was provided. This is a default message."
@@ -254,6 +257,11 @@ type Agent struct {
 	// tlsConfigurator is the central instance to provide a *tls.Config
 	// based on the current consul configuration.
 	tlsConfigurator *tlsutil.Configurator
+
+	// persistedTokensLock is used to synchronize access to the persisted token
+	// store within the data directory. This will prevent loading while writing as
+	// well as multiple concurrent writes.
+	persistedTokensLock sync.RWMutex
 }
 
 func New(c *config.RuntimeConfig) (*Agent, error) {
@@ -288,12 +296,6 @@ func New(c *config.RuntimeConfig) (*Agent, error) {
 		return nil, err
 	}
 
-	// Set up the initial state of the token store based on the config.
-	a.tokens.UpdateUserToken(a.config.ACLToken)
-	a.tokens.UpdateAgentToken(a.config.ACLAgentToken)
-	a.tokens.UpdateAgentMasterToken(a.config.ACLAgentMasterToken)
-	a.tokens.UpdateACLReplicationToken(a.config.ACLReplicationToken)
-
 	return a, nil
 }
 
@@ -367,6 +369,10 @@ func (a *Agent) Start() error {
 			"1 and 63 bytes.", a.config.NodeName)
 	}
 
+	// load the tokens - this requires the logger to be setup
+	// which is why we can't do this in New
+	a.loadTokens(a.config)
+
 	// create the local state
 	a.State = local.NewState(LocalConfig(c), a.logger, a.tokens)
 
@@ -590,6 +596,7 @@ func (a *Agent) listenAndServeDNS() error {
 		select {
 		case addr := <-notif:
 			a.logger.Printf("[INFO] agent: Started DNS server %s (%s)", addr.String(), addr.Network())
+
 		case err := <-errCh:
 			merr = multierror.Append(merr, err)
 		case <-timeout:
@@ -1072,7 +1079,6 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	// Copy the Connect CA bootstrap config
 	if a.config.ConnectEnabled {
 		base.ConnectEnabled = true
-		base.ConnectReplicationToken = a.config.ConnectReplicationToken
 
 		// Allow config to specify cluster_id provided it's a valid UUID. This is
 		// meant only for tests where a deterministic ID makes fixtures much simpler
@@ -3195,6 +3201,90 @@ func (a *Agent) loadProxies(conf *config.RuntimeConfig) error {
 	return persistenceErr
 }
 
+type persistedTokens struct {
+	Replication string `json:"replication,omitempty"`
+	AgentMaster string `json:"agent_master,omitempty"`
+	Default     string `json:"default,omitempty"`
+	Agent       string `json:"agent,omitempty"`
+}
+
+func (a *Agent) getPersistedTokens() (*persistedTokens, error) {
+	persistedTokens := &persistedTokens{}
+	if !a.config.ACLEnableTokenPersistence {
+		return persistedTokens, nil
+	}
+
+	a.persistedTokensLock.RLock()
+	defer a.persistedTokensLock.RUnlock()
+
+	tokensFullPath := filepath.Join(a.config.DataDir, tokensPath)
+
+	buf, err := ioutil.ReadFile(tokensFullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// non-existence is not an error we care about
+			return persistedTokens, nil
+		}
+		return persistedTokens, fmt.Errorf("failed reading tokens file %q: %s", tokensFullPath, err)
+	}
+
+	if err := json.Unmarshal(buf, persistedTokens); err != nil {
+		return persistedTokens, fmt.Errorf("failed to decode tokens file %q: %s", tokensFullPath, err)
+	}
+
+	return persistedTokens, nil
+}
+
+func (a *Agent) loadTokens(conf *config.RuntimeConfig) error {
+	persistedTokens, persistenceErr := a.getPersistedTokens()
+
+	if persistenceErr != nil {
+		a.logger.Printf("[WARN] unable to load persisted tokens: %v", persistenceErr)
+	}
+
+	if persistedTokens.Default != "" {
+		a.tokens.UpdateUserToken(persistedTokens.Default, token.TokenSourceAPI)
+
+		if conf.ACLToken != "" {
+			a.logger.Printf("[WARN] \"default\" token present in both the configuration and persisted token store, using the persisted token")
+		}
+	} else {
+		a.tokens.UpdateUserToken(conf.ACLToken, token.TokenSourceConfig)
+	}
+
+	if persistedTokens.Agent != "" {
+		a.tokens.UpdateAgentToken(persistedTokens.Agent, token.TokenSourceAPI)
+
+		if conf.ACLAgentToken != "" {
+			a.logger.Printf("[WARN] \"agent\" token present in both the configuration and persisted token store, using the persisted token")
+		}
+	} else {
+		a.tokens.UpdateAgentToken(conf.ACLAgentToken, token.TokenSourceConfig)
+	}
+
+	if persistedTokens.AgentMaster != "" {
+		a.tokens.UpdateAgentMasterToken(persistedTokens.AgentMaster, token.TokenSourceAPI)
+
+		if conf.ACLAgentMasterToken != "" {
+			a.logger.Printf("[WARN] \"agent_master\" token present in both the configuration and persisted token store, using the persisted token")
+		}
+	} else {
+		a.tokens.UpdateAgentMasterToken(conf.ACLAgentMasterToken, token.TokenSourceConfig)
+	}
+
+	if persistedTokens.Replication != "" {
+		a.tokens.UpdateReplicationToken(persistedTokens.Replication, token.TokenSourceAPI)
+
+		if conf.ACLReplicationToken != "" {
+			a.logger.Printf("[WARN] \"replication\" token present in both the configuration and persisted token store, using the persisted token")
+		}
+	} else {
+		a.tokens.UpdateReplicationToken(conf.ACLReplicationToken, token.TokenSourceConfig)
+	}
+
+	return persistenceErr
+}
+
 // unloadProxies will deregister all proxies known to the local agent.
 func (a *Agent) unloadProxies() error {
 	a.proxyLock.Lock()
@@ -3359,6 +3449,11 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error {
 	}
 	a.unloadMetadata()
 
+	// Reload tokens - should be done before all the other loading
+	// to ensure the correct tokens are available for attaching to
+	// the checks and service registrations.
+	a.loadTokens(newCfg)
+
 	// Reload service/check definitions and metadata.
 	if err := a.loadServices(newCfg); err != nil {
 		return fmt.Errorf("Failed reloading services: %s", err)

agent/agent_endpoint.go

@@ -1,11 +1,13 @@
 package agent
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
 	"net"
 	"net/http"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -22,9 +24,11 @@ import (
 	"github.com/hashicorp/consul/agent/debug"
 	"github.com/hashicorp/consul/agent/local"
 	"github.com/hashicorp/consul/agent/structs"
+	token_store "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/ipaddr"
 	"github.com/hashicorp/consul/lib"
+	"github.com/hashicorp/consul/lib/file"
 	"github.com/hashicorp/consul/logger"
 	"github.com/hashicorp/consul/types"
 	"github.com/hashicorp/logutils"
@@ -1262,23 +1266,32 @@ func (s *HTTPServer) AgentToken(resp http.ResponseWriter, req *http.Request) (in
 		return nil, nil
 	}
 
+	if s.agent.config.ACLEnableTokenPersistence {
+		// we hold the lock around updating the internal token store
+		// as well as persisting the tokens because we don't want to write
+		// into the store to have something else wipe it out before we can
+		// persist everything (like an agent config reload). The token store
+		// lock is only held for those operations so other go routines that
+		// just need to read some token out of the store will not be impacted
+		// any more than they would be without token persistence.
+		s.agent.persistedTokensLock.Lock()
+		defer s.agent.persistedTokensLock.Unlock()
+	}
+
 	// Figure out the target token.
 	target := strings.TrimPrefix(req.URL.Path, "/v1/agent/token/")
 	switch target {
-	case "acl_token":
+	case "acl_token", "default":
-		s.agent.tokens.UpdateUserToken(args.Token)
+		s.agent.tokens.UpdateUserToken(args.Token, token_store.TokenSourceAPI)
 
-	case "acl_agent_token":
+	case "acl_agent_token", "agent":
-		s.agent.tokens.UpdateAgentToken(args.Token)
+		s.agent.tokens.UpdateAgentToken(args.Token, token_store.TokenSourceAPI)
 
-	case "acl_agent_master_token":
+	case "acl_agent_master_token", "agent_master":
-		s.agent.tokens.UpdateAgentMasterToken(args.Token)
+		s.agent.tokens.UpdateAgentMasterToken(args.Token, token_store.TokenSourceAPI)
 
-	case "acl_replication_token":
+	case "acl_replication_token", "replication":
-		s.agent.tokens.UpdateACLReplicationToken(args.Token)
+		s.agent.tokens.UpdateReplicationToken(args.Token, token_store.TokenSourceAPI)
-
-	case "connect_replication_token":
-		s.agent.tokens.UpdateConnectReplicationToken(args.Token)
 
 	default:
 		resp.WriteHeader(http.StatusNotFound)
@@ -1286,6 +1299,37 @@ func (s *HTTPServer) AgentToken(resp http.ResponseWriter, req *http.Request) (in
 		return nil, nil
 	}
 
+	if s.agent.config.ACLEnableTokenPersistence {
+		tokens := persistedTokens{}
+
+		if tok, source := s.agent.tokens.UserTokenAndSource(); tok != "" && source == token_store.TokenSourceAPI {
+			tokens.Default = tok
+		}
+
+		if tok, source := s.agent.tokens.AgentTokenAndSource(); tok != "" && source == token_store.TokenSourceAPI {
+			tokens.Agent = tok
+		}
+
+		if tok, source := s.agent.tokens.AgentMasterTokenAndSource(); tok != "" && source == token_store.TokenSourceAPI {
+			tokens.AgentMaster = tok
+		}
+
+		if tok, source := s.agent.tokens.ReplicationTokenAndSource(); tok != "" && source == token_store.TokenSourceAPI {
+			tokens.Replication = tok
+		}
+
+		data, err := json.Marshal(tokens)
+		if err != nil {
+			s.agent.logger.Printf("[WARN] agent: failed to persist tokens - %v", err)
+			return nil, fmt.Errorf("Failed to marshal tokens for persistence: %v", err)
+		}
+
+		if err := file.WriteAtomicWithPerms(filepath.Join(s.agent.config.DataDir, tokensPath), data, 0600); err != nil {
+			s.agent.logger.Printf("[WARN] agent: failed to persist tokens - %v", err)
+			return nil, fmt.Errorf("Failed to persist tokens - %v", err)
+		}
+	}
+
 	s.agent.logger.Printf("[INFO] agent: Updated agent's ACL token %q", target)
 	return nil, nil
 }

agent/agent_endpoint_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/hashicorp/consul/agent/debug"
 	"github.com/hashicorp/consul/agent/local"
 	"github.com/hashicorp/consul/agent/structs"
+	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/logger"
@@ -4078,22 +4079,34 @@ func TestAgent_Token(t *testing.T) {
 	// in TestACL_Disabled_Response since there's already good infra set
 	// up over there to test this, and it calls the common function.
 	a := NewTestAgent(t, t.Name(), TestACLConfig()+`
-		acl_token = ""
+		acl {
-		acl_agent_token = ""
+			tokens {
-		acl_agent_master_token = ""
+				default = ""
+				agent = ""
+				agent_master = ""
+				replication = ""
+			}
+		}
 	`)
 	defer a.Shutdown()
 	testrpc.WaitForLeader(t, a.RPC, "dc1")
 
 	type tokens struct {
-		user, agent, master, repl string
+		user         string
+		userSource   tokenStore.TokenSource
+		agent        string
+		agentSource  tokenStore.TokenSource
+		master       string
+		masterSource tokenStore.TokenSource
+		repl         string
+		replSource   tokenStore.TokenSource
 	}
 
-	resetTokens := func(got tokens) {
+	resetTokens := func(init tokens) {
-		a.tokens.UpdateUserToken(got.user)
+		a.tokens.UpdateUserToken(init.user, init.userSource)
-		a.tokens.UpdateAgentToken(got.agent)
+		a.tokens.UpdateAgentToken(init.agent, init.agentSource)
-		a.tokens.UpdateAgentMasterToken(got.master)
+		a.tokens.UpdateAgentMasterToken(init.master, init.masterSource)
-		a.tokens.UpdateACLReplicationToken(got.repl)
+		a.tokens.UpdateReplicationToken(init.repl, init.replSource)
 	}
 
 	body := func(token string) io.Reader {
@@ -4109,7 +4122,9 @@ func TestAgent_Token(t *testing.T) {
 		method, url string
 		body        io.Reader
 		code        int
-		got, want   tokens
+		init        tokens
+		raw         tokens
+		effective   tokens
 	}{
 		{
 			name:   "bad token name",
@@ -4126,95 +4141,181 @@ func TestAgent_Token(t *testing.T) {
 			code:   http.StatusBadRequest,
 		},
 		{
-			name:   "set user",
+			name:      "set user legacy",
 			method:    "PUT",
 			url:       "acl_token?token=root",
 			body:      body("U"),
 			code:      http.StatusOK,
-			want:   tokens{user: "U", agent: "U"},
+			raw:       tokens{user: "U", userSource: tokenStore.TokenSourceAPI},
+			effective: tokens{user: "U", agent: "U"},
 		},
 		{
-			name:   "set agent",
+			name:      "set default",
+			method:    "PUT",
+			url:       "default?token=root",
+			body:      body("U"),
+			code:      http.StatusOK,
+			raw:       tokens{user: "U", userSource: tokenStore.TokenSourceAPI},
+			effective: tokens{user: "U", agent: "U"},
+		},
+		{
+			name:      "set agent legacy",
 			method:    "PUT",
 			url:       "acl_agent_token?token=root",
 			body:      body("A"),
 			code:      http.StatusOK,
-			got:    tokens{user: "U", agent: "U"},
+			init:      tokens{user: "U", agent: "U"},
-			want:   tokens{user: "U", agent: "A"},
+			raw:       tokens{user: "U", agent: "A", agentSource: tokenStore.TokenSourceAPI},
+			effective: tokens{user: "U", agent: "A"},
 		},
 		{
-			name:   "set master",
+			name:      "set agent",
+			method:    "PUT",
+			url:       "agent?token=root",
+			body:      body("A"),
+			code:      http.StatusOK,
+			init:      tokens{user: "U", agent: "U"},
+			raw:       tokens{user: "U", agent: "A", agentSource: tokenStore.TokenSourceAPI},
+			effective: tokens{user: "U", agent: "A"},
+		},
+		{
+			name:      "set master legacy",
 			method:    "PUT",
 			url:       "acl_agent_master_token?token=root",
 			body:      body("M"),
 			code:      http.StatusOK,
-			want:   tokens{master: "M"},
+			raw:       tokens{master: "M", masterSource: tokenStore.TokenSourceAPI},
+			effective: tokens{master: "M"},
 		},
 		{
-			name:   "set repl",
+			name:      "set master ",
+			method:    "PUT",
+			url:       "agent_master?token=root",
+			body:      body("M"),
+			code:      http.StatusOK,
+			raw:       tokens{master: "M", masterSource: tokenStore.TokenSourceAPI},
+			effective: tokens{master: "M"},
+		},
+		{
+			name:      "set repl legacy",
 			method:    "PUT",
 			url:       "acl_replication_token?token=root",
 			body:      body("R"),
 			code:      http.StatusOK,
-			want:   tokens{repl: "R"},
+			raw:       tokens{repl: "R", replSource: tokenStore.TokenSourceAPI},
+			effective: tokens{repl: "R"},
 		},
 		{
-			name:   "clear user",
+			name:      "set repl",
+			method:    "PUT",
+			url:       "replication?token=root",
+			body:      body("R"),
+			code:      http.StatusOK,
+			raw:       tokens{repl: "R", replSource: tokenStore.TokenSourceAPI},
+			effective: tokens{repl: "R"},
+		},
+		{
+			name:   "clear user legacy",
 			method: "PUT",
 			url:    "acl_token?token=root",
 			body:   body(""),
 			code:   http.StatusOK,
-			got:    tokens{user: "U"},
+			init:   tokens{user: "U"},
+			raw:    tokens{userSource: tokenStore.TokenSourceAPI},
 		},
 		{
-			name:   "clear agent",
+			name:   "clear default",
+			method: "PUT",
+			url:    "default?token=root",
+			body:   body(""),
+			code:   http.StatusOK,
+			init:   tokens{user: "U"},
+			raw:    tokens{userSource: tokenStore.TokenSourceAPI},
+		},
+		{
+			name:   "clear agent legacy",
 			method: "PUT",
 			url:    "acl_agent_token?token=root",
 			body:   body(""),
 			code:   http.StatusOK,
-			got:    tokens{agent: "A"},
+			init:   tokens{agent: "A"},
+			raw:    tokens{agentSource: tokenStore.TokenSourceAPI},
 		},
 		{
-			name:   "clear master",
+			name:   "clear agent",
+			method: "PUT",
+			url:    "agent?token=root",
+			body:   body(""),
+			code:   http.StatusOK,
+			init:   tokens{agent: "A"},
+			raw:    tokens{agentSource: tokenStore.TokenSourceAPI},
+		},
+		{
+			name:   "clear master legacy",
 			method: "PUT",
 			url:    "acl_agent_master_token?token=root",
 			body:   body(""),
 			code:   http.StatusOK,
-			got:    tokens{master: "M"},
+			init:   tokens{master: "M"},
+			raw:    tokens{masterSource: tokenStore.TokenSourceAPI},
 		},
 		{
-			name:   "clear repl",
+			name:   "clear master",
+			method: "PUT",
+			url:    "agent_master?token=root",
+			body:   body(""),
+			code:   http.StatusOK,
+			init:   tokens{master: "M"},
+			raw:    tokens{masterSource: tokenStore.TokenSourceAPI},
+		},
+		{
+			name:   "clear repl legacy",
 			method: "PUT",
 			url:    "acl_replication_token?token=root",
 			body:   body(""),
 			code:   http.StatusOK,
-			got:    tokens{repl: "R"},
+			init:   tokens{repl: "R"},
+			raw:    tokens{replSource: tokenStore.TokenSourceAPI},
+		},
+		{
+			name:   "clear repl",
+			method: "PUT",
+			url:    "replication?token=root",
+			body:   body(""),
+			code:   http.StatusOK,
+			init:   tokens{repl: "R"},
+			raw:    tokens{replSource: tokenStore.TokenSourceAPI},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			resetTokens(tt.got)
+			resetTokens(tt.init)
 			url := fmt.Sprintf("/v1/agent/token/%s", tt.url)
 			resp := httptest.NewRecorder()
 			req, _ := http.NewRequest(tt.method, url, tt.body)
-			if _, err := a.srv.AgentToken(resp, req); err != nil {
+			_, err := a.srv.AgentToken(resp, req)
-				t.Fatalf("err: %v", err)
+			require.NoError(t, err)
-			}
+			require.Equal(t, tt.code, resp.Code)
-			if got, want := resp.Code, tt.code; got != want {
+			require.Equal(t, tt.effective.user, a.tokens.UserToken())
-				t.Fatalf("got %d want %d", got, want)
+			require.Equal(t, tt.effective.agent, a.tokens.AgentToken())
-			}
+			require.Equal(t, tt.effective.master, a.tokens.AgentMasterToken())
-			if got, want := a.tokens.UserToken(), tt.want.user; got != want {
+			require.Equal(t, tt.effective.repl, a.tokens.ReplicationToken())
-				t.Fatalf("got %q want %q", got, want)
+
-			}
+			tok, src := a.tokens.UserTokenAndSource()
-			if got, want := a.tokens.AgentToken(), tt.want.agent; got != want {
+			require.Equal(t, tt.raw.user, tok)
-				t.Fatalf("got %q want %q", got, want)
+			require.Equal(t, tt.raw.userSource, src)
-			}
+
-			if tt.want.master != "" && !a.tokens.IsAgentMasterToken(tt.want.master) {
+			tok, src = a.tokens.AgentTokenAndSource()
-				t.Fatalf("%q should be the master token", tt.want.master)
+			require.Equal(t, tt.raw.agent, tok)
-			}
+			require.Equal(t, tt.raw.agentSource, src)
-			if got, want := a.tokens.ACLReplicationToken(), tt.want.repl; got != want {
+
-				t.Fatalf("got %q want %q", got, want)
+			tok, src = a.tokens.AgentMasterTokenAndSource()
-			}
+			require.Equal(t, tt.raw.master, tok)
+			require.Equal(t, tt.raw.masterSource, src)
+
+			tok, src = a.tokens.ReplicationTokenAndSource()
+			require.Equal(t, tt.raw.repl, tok)
+			require.Equal(t, tt.raw.replSource, src)
 		})
 	}
 
@@ -4223,12 +4324,9 @@ func TestAgent_Token(t *testing.T) {
 	t.Run("permission denied", func(t *testing.T) {
 		resetTokens(tokens{})
 		req, _ := http.NewRequest("PUT", "/v1/agent/token/acl_token", body("X"))
-		if _, err := a.srv.AgentToken(nil, req); !acl.IsErrPermissionDenied(err) {
+		_, err := a.srv.AgentToken(nil, req)
-			t.Fatalf("err: %v", err)
+		require.True(t, acl.IsErrPermissionDenied(err))
-		}
+		require.Equal(t, "", a.tokens.UserToken())
-		if got, want := a.tokens.UserToken(), ""; got != want {
-			t.Fatalf("got %q want %q", got, want)
-		}
 	})
 }
 

agent/agent_test.go

@@ -3386,3 +3386,160 @@ func TestAgent_SetupProxyManager(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, a.setupProxyManager())
 }
+
+func TestAgent_loadTokens(t *testing.T) {
+	t.Parallel()
+	a := NewTestAgent(t, t.Name(), `
+		acl = {
+			enabled = true
+			tokens = {
+				agent = "alfa"
+				agent_master = "bravo",
+				default = "charlie"
+				replication = "delta"
+			}
+		}
+
+	`)
+	defer a.Shutdown()
+	require := require.New(t)
+
+	tokensFullPath := filepath.Join(a.config.DataDir, tokensPath)
+
+	t.Run("original-configuration", func(t *testing.T) {
+		require.Equal("alfa", a.tokens.AgentToken())
+		require.Equal("bravo", a.tokens.AgentMasterToken())
+		require.Equal("charlie", a.tokens.UserToken())
+		require.Equal("delta", a.tokens.ReplicationToken())
+	})
+
+	t.Run("updated-configuration", func(t *testing.T) {
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "echo",
+			ACLAgentToken:       "foxtrot",
+			ACLAgentMasterToken: "golf",
+			ACLReplicationToken: "hotel",
+		}
+		// ensures no error for missing persisted tokens file
+		require.NoError(a.loadTokens(cfg))
+		require.Equal("echo", a.tokens.UserToken())
+		require.Equal("foxtrot", a.tokens.AgentToken())
+		require.Equal("golf", a.tokens.AgentMasterToken())
+		require.Equal("hotel", a.tokens.ReplicationToken())
+	})
+
+	t.Run("persisted-tokens", func(t *testing.T) {
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "echo",
+			ACLAgentToken:       "foxtrot",
+			ACLAgentMasterToken: "golf",
+			ACLReplicationToken: "hotel",
+		}
+
+		tokens := `{
+			"agent" : "india",
+			"agent_master" : "juliett",
+			"default": "kilo",
+			"replication" : "lima"
+		}`
+
+		require.NoError(ioutil.WriteFile(tokensFullPath, []byte(tokens), 0600))
+		require.NoError(a.loadTokens(cfg))
+
+		// no updates since token persistence is not enabled
+		require.Equal("echo", a.tokens.UserToken())
+		require.Equal("foxtrot", a.tokens.AgentToken())
+		require.Equal("golf", a.tokens.AgentMasterToken())
+		require.Equal("hotel", a.tokens.ReplicationToken())
+
+		a.config.ACLEnableTokenPersistence = true
+		require.NoError(a.loadTokens(cfg))
+
+		require.Equal("india", a.tokens.AgentToken())
+		require.Equal("juliett", a.tokens.AgentMasterToken())
+		require.Equal("kilo", a.tokens.UserToken())
+		require.Equal("lima", a.tokens.ReplicationToken())
+	})
+
+	t.Run("persisted-tokens-override", func(t *testing.T) {
+		tokens := `{
+			"agent" : "mike",
+			"agent_master" : "november",
+			"default": "oscar",
+			"replication" : "papa"
+		}`
+
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "quebec",
+			ACLAgentToken:       "romeo",
+			ACLAgentMasterToken: "sierra",
+			ACLReplicationToken: "tango",
+		}
+
+		require.NoError(ioutil.WriteFile(tokensFullPath, []byte(tokens), 0600))
+		require.NoError(a.loadTokens(cfg))
+
+		require.Equal("mike", a.tokens.AgentToken())
+		require.Equal("november", a.tokens.AgentMasterToken())
+		require.Equal("oscar", a.tokens.UserToken())
+		require.Equal("papa", a.tokens.ReplicationToken())
+	})
+
+	t.Run("partial-persisted", func(t *testing.T) {
+		tokens := `{
+			"agent" : "uniform",
+			"agent_master" : "victor"
+		}`
+
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "whiskey",
+			ACLAgentToken:       "xray",
+			ACLAgentMasterToken: "yankee",
+			ACLReplicationToken: "zulu",
+		}
+
+		require.NoError(ioutil.WriteFile(tokensFullPath, []byte(tokens), 0600))
+		require.NoError(a.loadTokens(cfg))
+
+		require.Equal("uniform", a.tokens.AgentToken())
+		require.Equal("victor", a.tokens.AgentMasterToken())
+		require.Equal("whiskey", a.tokens.UserToken())
+		require.Equal("zulu", a.tokens.ReplicationToken())
+	})
+
+	t.Run("persistence-error-not-json", func(t *testing.T) {
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "one",
+			ACLAgentToken:       "two",
+			ACLAgentMasterToken: "three",
+			ACLReplicationToken: "four",
+		}
+
+		require.NoError(ioutil.WriteFile(tokensFullPath, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, 0600))
+		err := a.loadTokens(cfg)
+		require.Error(err)
+
+		require.Equal("one", a.tokens.UserToken())
+		require.Equal("two", a.tokens.AgentToken())
+		require.Equal("three", a.tokens.AgentMasterToken())
+		require.Equal("four", a.tokens.ReplicationToken())
+	})
+
+	t.Run("persistence-error-wrong-top-level", func(t *testing.T) {
+		cfg := &config.RuntimeConfig{
+			ACLToken:            "alfa",
+			ACLAgentToken:       "bravo",
+			ACLAgentMasterToken: "charlie",
+			ACLReplicationToken: "foxtrot",
+		}
+
+		require.NoError(ioutil.WriteFile(tokensFullPath, []byte("[1,2,3]"), 0600))
+		err := a.loadTokens(cfg)
+		require.Error(err)
+
+		require.Equal("alfa", a.tokens.UserToken())
+		require.Equal("bravo", a.tokens.AgentToken())
+		require.Equal("charlie", a.tokens.AgentMasterToken())
+		require.Equal("foxtrot", a.tokens.ReplicationToken())
+	})
+}

agent/config/builder.go

@@ -688,6 +688,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		ACLPolicyTTL:              b.durationVal("acl.policy_ttl", c.ACL.PolicyTTL),
 		ACLToken:                  b.stringValWithDefault(c.ACL.Tokens.Default, b.stringVal(c.ACLToken)),
 		ACLTokenReplication:       b.boolValWithDefault(c.ACL.TokenReplication, b.boolValWithDefault(c.EnableACLReplication, enableTokenReplication)),
+		ACLEnableTokenPersistence: b.boolValWithDefault(c.ACL.EnableTokenPersistence, false),
 
 		// Autopilot
 		AutopilotCleanupDeadServers:      b.boolVal(c.Autopilot.CleanupDeadServers),
@@ -779,7 +780,6 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		ConnectProxyDefaultDaemonCommand:        proxyDefaultDaemonCommand,
 		ConnectProxyDefaultScriptCommand:        proxyDefaultScriptCommand,
 		ConnectProxyDefaultConfig:               proxyDefaultConfig,
-		ConnectReplicationToken:                 b.stringVal(c.ACL.Tokens.Replication),
 		DataDir:                                 b.stringVal(c.DataDir),
 		Datacenter:                              datacenter,
 		DevMode:                                 b.boolVal(b.Flags.DevMode),

agent/config/config.go

@@ -639,6 +639,7 @@ type ACL struct {
 	EnableKeyListPolicy    *bool   `json:"enable_key_list_policy,omitempty" hcl:"enable_key_list_policy" mapstructure:"enable_key_list_policy"`
 	Tokens                 Tokens  `json:"tokens,omitempty" hcl:"tokens" mapstructure:"tokens"`
 	DisabledTTL            *string `json:"disabled_ttl,omitempty" hcl:"disabled_ttl" mapstructure:"disabled_ttl"`
+	EnableTokenPersistence *bool   `json:"enable_token_persistence" hcl:"enable_token_persistence" mapstructure:"enable_token_persistence"`
 }
 
 type Tokens struct {

agent/config/runtime.go

@@ -127,10 +127,12 @@ type RuntimeConfig struct {
 	// hcl: acl.tokens.master = string
 	ACLMasterToken string
 
-	// ACLReplicationToken is used to fetch ACLs from the ACLDatacenter in
+	// ACLReplicationToken is used to replicate data locally from the
-	// order to replicate them locally. Setting this to a non-empty value
+	// PrimaryDatacenter. Replication is only available on servers in
-	// also enables replication. Replication is only available in datacenters
+	// datacenters other than the PrimaryDatacenter
-	// other than the ACLDatacenter.
+	//
+	// DEPRECATED (ACL-Legacy-Compat): Setting this to a non-empty value
+	// also enables legacy ACL replication if ACLs are enabled and in legacy mode.
 	//
 	// hcl: acl.tokens.replication = string
 	ACLReplicationToken string
@@ -159,6 +161,10 @@ type RuntimeConfig struct {
 	// hcl: acl.tokens.default = string
 	ACLToken string
 
+	// ACLEnableTokenPersistence determines whether or not tokens set via the agent HTTP API
+	// should be persisted to disk and reloaded when an agent restarts.
+	ACLEnableTokenPersistence bool
+
 	// AutopilotCleanupDeadServers enables the automatic cleanup of dead servers when new ones
 	// are added to the peer list. Defaults to true.
 	//
@@ -545,9 +551,6 @@ type RuntimeConfig struct {
 	// ConnectCAConfig is the config to use for the CA provider.
 	ConnectCAConfig map[string]interface{}
 
-	// ConnectReplicationToken is the ACL token used for replicating intentions.
-	ConnectReplicationToken string
-
 	// ConnectTestDisableManagedProxies is not exposed to public config but is
 	// used by TestAgent to prevent self-executing the test binary in the
 	// background if a managed proxy is created for a test. The only place we

agent/config/runtime_test.go

@@ -2899,6 +2899,7 @@ func TestFullConfig(t *testing.T) {
 				"down_policy" : "03eb2aee",
 				"default_policy" : "72c2e7a0",
 				"enable_key_list_policy": false,
+				"enable_token_persistence": true,
 				"policy_ttl": "1123s",
 				"token_ttl": "3321s",
 				"enable_token_replication" : true,
@@ -3450,6 +3451,7 @@ func TestFullConfig(t *testing.T) {
 				down_policy = "03eb2aee"
 				default_policy = "72c2e7a0"
 				enable_key_list_policy = false
+				enable_token_persistence = true
 				policy_ttl = "1123s"
 				token_ttl = "3321s"
 				enable_token_replication = true
@@ -4120,6 +4122,7 @@ func TestFullConfig(t *testing.T) {
 		ACLDownPolicy:                    "03eb2aee",
 		ACLEnforceVersion8:               true,
 		ACLEnableKeyListPolicy:           false,
+		ACLEnableTokenPersistence:        true,
 		ACLMasterToken:                   "8a19ac27",
 		ACLReplicationToken:              "5795983a",
 		ACLTokenTTL:                      3321 * time.Second,
@@ -4236,7 +4239,6 @@ func TestFullConfig(t *testing.T) {
 			"connect_timeout_ms": float64(1000),
 			"pedantic_mode":      true,
 		},
-		ConnectReplicationToken:          "5795983a",
 		DNSAddrs:                         []net.Addr{tcpAddr("93.95.95.81:7001"), udpAddr("93.95.95.81:7001")},
 		DNSARecordLimit:                  29907,
 		DNSAllowStale:                    true,
@@ -4938,6 +4940,7 @@ func TestSanitize(t *testing.T) {
 		"ACLDisabledTTL": "0s",
 		"ACLDownPolicy": "",
 		"ACLEnableKeyListPolicy": false,
+		"ACLEnableTokenPersistence": false,
 		"ACLEnforceVersion8": false,
 		"ACLMasterToken": "hidden",
 		"ACLPolicyTTL": "0s",
@@ -5004,7 +5007,6 @@ func TestSanitize(t *testing.T) {
 		"ConnectSidecarMaxPort": 0,
 		"ConnectSidecarMinPort": 0,
 		"ConnectTestCALeafRootChangeSpread": "0s",
-		"ConnectReplicationToken": "hidden",
 		"ConnectTestDisableManagedProxies": false,
 		"ConsulCoordinateUpdateBatchSize": 0,
 		"ConsulCoordinateUpdateMaxBatches": 0,

agent/consul/acl_endpoint_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/structs"
+	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/testrpc"
 	"github.com/hashicorp/consul/testutil/retry"
@@ -598,7 +599,7 @@ func TestACLEndpoint_ReplicationStatus(t *testing.T) {
 		c.ACLReplicationRate = 100
 		c.ACLReplicationBurst = 100
 	})
-	s1.tokens.UpdateACLReplicationToken("secret")
+	s1.tokens.UpdateReplicationToken("secret", tokenStore.TokenSourceConfig)
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
 	codec := rpcClient(t, s1)
@@ -876,7 +877,7 @@ func TestACLEndpoint_TokenDelete(t *testing.T) {
 	codec2 := rpcClient(t, s2)
 	defer codec2.Close()
 
-	s2.tokens.UpdateACLReplicationToken("root")
+	s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
 
 	testrpc.WaitForLeader(t, s1.RPC, "dc1")
 	testrpc.WaitForLeader(t, s2.RPC, "dc2")

agent/consul/acl_replication.go

@@ -140,7 +140,7 @@ func (s *Server) fetchACLPoliciesBatch(policyIDs []string) (*structs.ACLPolicyBa
 		PolicyIDs:  policyIDs,
 		QueryOptions: structs.QueryOptions{
 			AllowStale: true,
-			Token:      s.tokens.ACLReplicationToken(),
+			Token:      s.tokens.ReplicationToken(),
 		},
 	}
 
@@ -160,7 +160,7 @@ func (s *Server) fetchACLPolicies(lastRemoteIndex uint64) (*structs.ACLPolicyLis
 		QueryOptions: structs.QueryOptions{
 			AllowStale:    true,
 			MinQueryIndex: lastRemoteIndex,
-			Token:         s.tokens.ACLReplicationToken(),
+			Token:         s.tokens.ReplicationToken(),
 		},
 	}
 
@@ -323,7 +323,7 @@ func (s *Server) fetchACLTokensBatch(tokenIDs []string) (*structs.ACLTokenBatchR
 		AccessorIDs: tokenIDs,
 		QueryOptions: structs.QueryOptions{
 			AllowStale: true,
-			Token:      s.tokens.ACLReplicationToken(),
+			Token:      s.tokens.ReplicationToken(),
 		},
 	}
 
@@ -343,7 +343,7 @@ func (s *Server) fetchACLTokens(lastRemoteIndex uint64) (*structs.ACLTokenListRe
 		QueryOptions: structs.QueryOptions{
 			AllowStale:    true,
 			MinQueryIndex: lastRemoteIndex,
-			Token:         s.tokens.ACLReplicationToken(),
+			Token:         s.tokens.ReplicationToken(),
 		},
 		IncludeLocal:  false,
 		IncludeGlobal: true,

agent/consul/acl_replication_legacy.go

@@ -162,7 +162,7 @@ func (s *Server) fetchRemoteLegacyACLs(lastRemoteIndex uint64) (*structs.Indexed
 	args := structs.DCSpecificRequest{
 		Datacenter: s.config.ACLDatacenter,
 		QueryOptions: structs.QueryOptions{
-			Token:         s.tokens.ACLReplicationToken(),
+			Token:         s.tokens.ReplicationToken(),
 			MinQueryIndex: lastRemoteIndex,
 			AllowStale:    true,
 		},

agent/consul/acl_replication_legacy_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/agent/structs"
+	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/testrpc"
 	"github.com/hashicorp/consul/testutil/retry"
 )
@@ -233,7 +234,7 @@ func TestACLReplication_updateLocalACLs_RateLimit(t *testing.T) {
 		c.ACLsEnabled = true
 		c.ACLReplicationApplyLimit = 1
 	})
-	s1.tokens.UpdateACLReplicationToken("secret")
+	s1.tokens.UpdateReplicationToken("secret", tokenStore.TokenSourceConfig)
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
 	testrpc.WaitForLeader(t, s1.RPC, "dc2")
@@ -356,7 +357,7 @@ func TestACLReplication_LegacyTokens(t *testing.T) {
 		c.ACLReplicationBurst = 100
 		c.ACLReplicationApplyLimit = 1000000
 	})
-	s2.tokens.UpdateACLReplicationToken("root")
+	s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
 	testrpc.WaitForLeader(t, s2.RPC, "dc2")
 	defer os.RemoveAll(dir2)
 	defer s2.Shutdown()

agent/consul/acl_replication_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/structs"
+	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/testrpc"
 	"github.com/hashicorp/consul/testutil/retry"
 	"github.com/stretchr/testify/require"
@@ -295,7 +296,7 @@ func TestACLReplication_Tokens(t *testing.T) {
 		c.ACLReplicationBurst = 100
 		c.ACLReplicationApplyLimit = 1000000
 	})
-	s2.tokens.UpdateACLReplicationToken("root")
+	s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
 	testrpc.WaitForLeader(t, s2.RPC, "dc2")
 	defer os.RemoveAll(dir2)
 	defer s2.Shutdown()
@@ -467,7 +468,7 @@ func TestACLReplication_Policies(t *testing.T) {
 		c.ACLReplicationBurst = 100
 		c.ACLReplicationApplyLimit = 1000000
 	})
-	s2.tokens.UpdateACLReplicationToken("root")
+	s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
 	testrpc.WaitForLeader(t, s2.RPC, "dc2")
 	defer os.RemoveAll(dir2)
 	defer s2.Shutdown()

agent/consul/acl_server.go

@@ -110,7 +110,7 @@ func (s *Server) LocalTokensEnabled() bool {
 		return true
 	}
 
-	if !s.config.ACLTokenReplication || s.tokens.ACLReplicationToken() == "" {
+	if !s.config.ACLTokenReplication || s.tokens.ReplicationToken() == "" {
 		return false
 	}
 

agent/consul/acl_test.go

@@ -1544,7 +1544,7 @@ func TestACL_Replication(t *testing.T) {
 			c.ACLReplicationBurst = 100
 			c.ACLReplicationApplyLimit = 1000000
 		})
-		s2.tokens.UpdateACLReplicationToken("root")
+		s2.tokens.UpdateReplicationToken("root")
 		defer os.RemoveAll(dir2)
 		defer s2.Shutdown()
 
@@ -1557,7 +1557,7 @@ func TestACL_Replication(t *testing.T) {
 			c.ACLReplicationBurst = 100
 			c.ACLReplicationApplyLimit = 1000000
 		})
-		s3.tokens.UpdateACLReplicationToken("root")
+		s3.tokens.UpdateReplicationToken("root")
 		defer os.RemoveAll(dir3)
 		defer s3.Shutdown()
 

agent/consul/config.go

@@ -377,9 +377,6 @@ type Config struct {
 	// CAConfig is used to apply the initial Connect CA configuration when
 	// bootstrapping.
 	CAConfig *structs.CAConfiguration
-
-	// ConnectReplicationToken is used to control Intention replication.
-	ConnectReplicationToken string
 }
 
 func (c *Config) ToTLSUtilConfig() *tlsutil.Config {

agent/consul/leader.go

@@ -684,7 +684,7 @@ func (s *Server) startLegacyACLReplication() {
 				return
 			}
 
-			if s.tokens.ACLReplicationToken() == "" {
+			if s.tokens.ReplicationToken() == "" {
 				continue
 			}
 
@@ -733,7 +733,7 @@ func (s *Server) startACLReplication() {
 				return
 			}
 
-			if s.tokens.ACLReplicationToken() == "" {
+			if s.tokens.ReplicationToken() == "" {
 				continue
 			}
 
@@ -779,7 +779,7 @@ func (s *Server) startACLReplication() {
 					return
 				}
 
-				if s.tokens.ACLReplicationToken() == "" {
+				if s.tokens.ReplicationToken() == "" {
 					continue
 				}
 

agent/http_test.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/agent/structs"
+	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/testrpc"
 	"github.com/hashicorp/consul/testutil"
@@ -1014,14 +1015,14 @@ func TestACLResolution(t *testing.T) {
 	defer a.Shutdown()
 
 	// Check when no token is set
-	a.tokens.UpdateUserToken("")
+	a.tokens.UpdateUserToken("", tokenStore.TokenSourceConfig)
 	a.srv.parseToken(req, &token)
 	if token != "" {
 		t.Fatalf("bad: %s", token)
 	}
 
 	// Check when ACLToken set
-	a.tokens.UpdateUserToken("agent")
+	a.tokens.UpdateUserToken("agent", tokenStore.TokenSourceAPI)
 	a.srv.parseToken(req, &token)
 	if token != "agent" {
 		t.Fatalf("bad: %s", token)

agent/local/state_test.go

@@ -1646,7 +1646,7 @@ func TestAgent_ServiceTokens(t *testing.T) {
 	t.Parallel()
 
 	tokens := new(token.Store)
-	tokens.UpdateUserToken("default")
+	tokens.UpdateUserToken("default", token.TokenSourceConfig)
 	cfg := config.DefaultRuntimeConfig(`bind_addr = "127.0.0.1" data_dir = "dummy"`)
 	l := local.NewState(agent.LocalConfig(cfg), nil, tokens)
 	l.TriggerSyncChanges = func() {}
@@ -1675,7 +1675,7 @@ func TestAgent_CheckTokens(t *testing.T) {
 	t.Parallel()
 
 	tokens := new(token.Store)
-	tokens.UpdateUserToken("default")
+	tokens.UpdateUserToken("default", token.TokenSourceConfig)
 	cfg := config.DefaultRuntimeConfig(`bind_addr = "127.0.0.1" data_dir = "dummy"`)
 	l := local.NewState(agent.LocalConfig(cfg), nil, tokens)
 	l.TriggerSyncChanges = func() {}

agent/token/store.go

@@ -4,6 +4,13 @@ import (
 	"sync"
 )
 
+type TokenSource bool
+
+const (
+	TokenSourceConfig TokenSource = false
+	TokenSourceAPI    TokenSource = true
+)
+
 // Store is used to hold the special ACL tokens used by Consul agents. It is
 // designed to update the tokens on the fly, so the token store itself should be
 // plumbed around and used to get tokens at runtime, don't save the resulting
@@ -17,57 +24,62 @@ type Store struct {
 	// also be used for agent operations if the agent token isn't set.
 	userToken string
 
+	// userTokenSource indicates where this token originated from
+	userTokenSource TokenSource
+
 	// agentToken is used for internal agent operations like self-registering
 	// with the catalog and anti-entropy, but should never be used for
 	// user-initiated operations.
 	agentToken string
 
+	// agentTokenSource indicates where this token originated from
+	agentTokenSource TokenSource
+
 	// agentMasterToken is a special token that's only used locally for
 	// access to the /v1/agent utility operations if the servers aren't
 	// available.
 	agentMasterToken string
 
-	// aclReplicationToken is a special token that's used by servers to
+	// agentMasterTokenSource indicates where this token originated from
-	// replicate ACLs from the ACL datacenter.
+	agentMasterTokenSource TokenSource
-	aclReplicationToken string
 
-	// connectReplicationToken is a special token that's used by servers to
+	// replicationToken is a special token that's used by servers to
-	// replicate intentions from the primary datacenter.
+	// replicate data from the primary datacenter.
-	connectReplicationToken string
+	replicationToken string
+
+	// replicationTokenSource indicates where this token originated from
+	replicationTokenSource TokenSource
 }
 
 // UpdateUserToken replaces the current user token in the store.
-func (t *Store) UpdateUserToken(token string) {
+func (t *Store) UpdateUserToken(token string, source TokenSource) {
 	t.l.Lock()
 	t.userToken = token
+	t.userTokenSource = source
 	t.l.Unlock()
 }
 
 // UpdateAgentToken replaces the current agent token in the store.
-func (t *Store) UpdateAgentToken(token string) {
+func (t *Store) UpdateAgentToken(token string, source TokenSource) {
 	t.l.Lock()
 	t.agentToken = token
+	t.agentTokenSource = source
 	t.l.Unlock()
 }
 
 // UpdateAgentMasterToken replaces the current agent master token in the store.
-func (t *Store) UpdateAgentMasterToken(token string) {
+func (t *Store) UpdateAgentMasterToken(token string, source TokenSource) {
 	t.l.Lock()
 	t.agentMasterToken = token
+	t.agentMasterTokenSource = source
 	t.l.Unlock()
 }
 
-// UpdateACLReplicationToken replaces the current ACL replication token in the store.
+// UpdateReplicationToken replaces the current replication token in the store.
-func (t *Store) UpdateACLReplicationToken(token string) {
+func (t *Store) UpdateReplicationToken(token string, source TokenSource) {
 	t.l.Lock()
-	t.aclReplicationToken = token
+	t.replicationToken = token
-	t.l.Unlock()
+	t.replicationTokenSource = source
-}
-
-// UpdateConnectReplicationToken replaces the current Connect replication token in the store.
-func (t *Store) UpdateConnectReplicationToken(token string) {
-	t.l.Lock()
-	t.connectReplicationToken = token
 	t.l.Unlock()
 }
 
@@ -90,20 +102,50 @@ func (t *Store) AgentToken() string {
 	return t.userToken
 }
 
-// ACLReplicationToken returns the ACL replication token.
+func (t *Store) AgentMasterToken() string {
-func (t *Store) ACLReplicationToken() string {
 	t.l.RLock()
 	defer t.l.RUnlock()
 
-	return t.aclReplicationToken
+	return t.agentMasterToken
 }
 
-// ConnectReplicationToken returns the Connect replication token.
+// ReplicationToken returns the replication token.
-func (t *Store) ConnectReplicationToken() string {
+func (t *Store) ReplicationToken() string {
 	t.l.RLock()
 	defer t.l.RUnlock()
 
-	return t.connectReplicationToken
+	return t.replicationToken
+}
+
+// UserToken returns the best token to use for user operations.
+func (t *Store) UserTokenAndSource() (string, TokenSource) {
+	t.l.RLock()
+	defer t.l.RUnlock()
+
+	return t.userToken, t.userTokenSource
+}
+
+// AgentToken returns the best token to use for internal agent operations.
+func (t *Store) AgentTokenAndSource() (string, TokenSource) {
+	t.l.RLock()
+	defer t.l.RUnlock()
+
+	return t.agentToken, t.agentTokenSource
+}
+
+func (t *Store) AgentMasterTokenAndSource() (string, TokenSource) {
+	t.l.RLock()
+	defer t.l.RUnlock()
+
+	return t.agentMasterToken, t.agentMasterTokenSource
+}
+
+// ReplicationToken returns the replication token.
+func (t *Store) ReplicationTokenAndSource() (string, TokenSource) {
+	t.l.RLock()
+	defer t.l.RUnlock()
+
+	return t.replicationToken, t.replicationTokenSource
 }
 
 // IsAgentMasterToken checks to see if a given token is the agent master token.

agent/token/store_test.go

@@ -2,60 +2,121 @@ package token
 
 import (
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestStore_RegularTokens(t *testing.T) {
 	t.Parallel()
 
 	type tokens struct {
-		user, agent, repl string
+		userSource   TokenSource
+		user         string
+		agent        string
+		agentSource  TokenSource
+		master       string
+		masterSource TokenSource
+		repl         string
+		replSource   TokenSource
 	}
 
 	tests := []struct {
 		name      string
-		set, want tokens
+		set       tokens
+		raw       tokens
+		effective tokens
 	}{
 		{
-			name: "set user",
+			name:      "set user - config",
-			set:  tokens{user: "U"},
+			set:       tokens{user: "U", userSource: TokenSourceConfig},
-			want: tokens{user: "U", agent: "U"},
+			raw:       tokens{user: "U", userSource: TokenSourceConfig},
+			effective: tokens{user: "U", agent: "U"},
 		},
 		{
-			name: "set agent",
+			name:      "set user - api",
-			set:  tokens{agent: "A"},
+			set:       tokens{user: "U", userSource: TokenSourceAPI},
-			want: tokens{agent: "A"},
+			raw:       tokens{user: "U", userSource: TokenSourceAPI},
+			effective: tokens{user: "U", agent: "U"},
+		},
+		{
+			name:      "set agent - config",
+			set:       tokens{agent: "A", agentSource: TokenSourceConfig},
+			raw:       tokens{agent: "A", agentSource: TokenSourceConfig},
+			effective: tokens{agent: "A"},
+		},
+		{
+			name:      "set agent - api",
+			set:       tokens{agent: "A", agentSource: TokenSourceAPI},
+			raw:       tokens{agent: "A", agentSource: TokenSourceAPI},
+			effective: tokens{agent: "A"},
 		},
 		{
 			name:      "set user and agent",
 			set:       tokens{agent: "A", user: "U"},
-			want: tokens{agent: "A", user: "U"},
+			raw:       tokens{agent: "A", user: "U"},
+			effective: tokens{agent: "A", user: "U"},
 		},
 		{
-			name: "set repl",
+			name:      "set repl - config",
-			set:  tokens{repl: "R"},
+			set:       tokens{repl: "R", replSource: TokenSourceConfig},
-			want: tokens{repl: "R"},
+			raw:       tokens{repl: "R", replSource: TokenSourceConfig},
+			effective: tokens{repl: "R"},
+		},
+		{
+			name:      "set repl - api",
+			set:       tokens{repl: "R", replSource: TokenSourceAPI},
+			raw:       tokens{repl: "R", replSource: TokenSourceAPI},
+			effective: tokens{repl: "R"},
+		},
+		{
+			name:      "set master - config",
+			set:       tokens{master: "M", masterSource: TokenSourceConfig},
+			raw:       tokens{master: "M", masterSource: TokenSourceConfig},
+			effective: tokens{master: "M"},
+		},
+		{
+			name:      "set master - api",
+			set:       tokens{master: "M", masterSource: TokenSourceAPI},
+			raw:       tokens{master: "M", masterSource: TokenSourceAPI},
+			effective: tokens{master: "M"},
 		},
 		{
 			name:      "set all",
-			set:  tokens{user: "U", agent: "A", repl: "R"},
+			set:       tokens{user: "U", agent: "A", repl: "R", master: "M"},
-			want: tokens{user: "U", agent: "A", repl: "R"},
+			raw:       tokens{user: "U", agent: "A", repl: "R", master: "M"},
+			effective: tokens{user: "U", agent: "A", repl: "R", master: "M"},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
 			s := new(Store)
-			s.UpdateUserToken(tt.set.user)
+			s.UpdateUserToken(tt.set.user, tt.set.userSource)
-			s.UpdateAgentToken(tt.set.agent)
+			s.UpdateAgentToken(tt.set.agent, tt.set.agentSource)
-			s.UpdateACLReplicationToken(tt.set.repl)
+			s.UpdateReplicationToken(tt.set.repl, tt.set.replSource)
-			if got, want := s.UserToken(), tt.want.user; got != want {
+			s.UpdateAgentMasterToken(tt.set.master, tt.set.masterSource)
-				t.Fatalf("got token %q want %q", got, want)
+
-			}
+			require.Equal(t, tt.effective.user, s.UserToken())
-			if got, want := s.AgentToken(), tt.want.agent; got != want {
+			require.Equal(t, tt.effective.agent, s.AgentToken())
-				t.Fatalf("got token %q want %q", got, want)
+			require.Equal(t, tt.effective.master, s.AgentMasterToken())
-			}
+			require.Equal(t, tt.effective.repl, s.ReplicationToken())
-			if got, want := s.ACLReplicationToken(), tt.want.repl; got != want {
+
-				t.Fatalf("got token %q want %q", got, want)
+			tok, src := s.UserTokenAndSource()
-			}
+			require.Equal(t, tt.raw.user, tok)
+			require.Equal(t, tt.raw.userSource, src)
+
+			tok, src = s.AgentTokenAndSource()
+			require.Equal(t, tt.raw.agent, tok)
+			require.Equal(t, tt.raw.agentSource, src)
+
+			tok, src = s.AgentMasterTokenAndSource()
+			require.Equal(t, tt.raw.master, tok)
+			require.Equal(t, tt.raw.masterSource, src)
+
+			tok, src = s.ReplicationTokenAndSource()
+			require.Equal(t, tt.raw.repl, tok)
+			require.Equal(t, tt.raw.replSource, src)
 		})
 	}
 }
@@ -66,22 +127,20 @@ func TestStore_AgentMasterToken(t *testing.T) {
 
 	verify := func(want bool, toks ...string) {
 		for _, tok := range toks {
-			if got := s.IsAgentMasterToken(tok); got != want {
+			require.Equal(t, want, s.IsAgentMasterToken(tok))
-				t.Fatalf("token %q got %v want %v", tok, got, want)
-			}
 		}
 	}
 
 	verify(false, "", "nope")
 
-	s.UpdateAgentMasterToken("master")
+	s.UpdateAgentMasterToken("master", TokenSourceConfig)
 	verify(true, "master")
 	verify(false, "", "nope")
 
-	s.UpdateAgentMasterToken("another")
+	s.UpdateAgentMasterToken("another", TokenSourceConfig)
 	verify(true, "another")
 	verify(false, "", "nope", "master")
 
-	s.UpdateAgentMasterToken("")
+	s.UpdateAgentMasterToken("", TokenSourceConfig)
 	verify(false, "", "nope", "master", "another")
 }

api/agent.go

@@ -926,41 +926,86 @@ func (a *Agent) Monitor(loglevel string, stopCh <-chan struct{}, q *QueryOptions
 
 // UpdateACLToken updates the agent's "acl_token". See updateToken for more
 // details.
+//
+// DEPRECATED (ACL-Legacy-Compat) - Prefer UpdateDefaultACLToken for v1.4.3 and above
 func (a *Agent) UpdateACLToken(token string, q *WriteOptions) (*WriteMeta, error) {
 	return a.updateToken("acl_token", token, q)
 }
 
 // UpdateACLAgentToken updates the agent's "acl_agent_token". See updateToken
 // for more details.
+//
+// DEPRECATED (ACL-Legacy-Compat) - Prefer UpdateAgentACLToken for v1.4.3 and above
 func (a *Agent) UpdateACLAgentToken(token string, q *WriteOptions) (*WriteMeta, error) {
 	return a.updateToken("acl_agent_token", token, q)
 }
 
 // UpdateACLAgentMasterToken updates the agent's "acl_agent_master_token". See
 // updateToken for more details.
+//
+// DEPRECATED (ACL-Legacy-Compat) - Prefer UpdateAgentMasterACLToken for v1.4.3 and above
 func (a *Agent) UpdateACLAgentMasterToken(token string, q *WriteOptions) (*WriteMeta, error) {
 	return a.updateToken("acl_agent_master_token", token, q)
 }
 
 // UpdateACLReplicationToken updates the agent's "acl_replication_token". See
 // updateToken for more details.
+//
+// DEPRECATED (ACL-Legacy-Compat) - Prefer UpdateReplicationACLToken for v1.4.3 and above
 func (a *Agent) UpdateACLReplicationToken(token string, q *WriteOptions) (*WriteMeta, error) {
 	return a.updateToken("acl_replication_token", token, q)
 }
 
-// updateToken can be used to update an agent's ACL token after the agent has
+// UpdateDefaultACLToken updates the agent's "default" token. See updateToken
-// started. The tokens are not persisted, so will need to be updated again if
+// for more details
-// the agent is restarted.
+func (a *Agent) UpdateDefaultACLToken(token string, q *WriteOptions) (*WriteMeta, error) {
+	return a.updateTokenFallback("default", "acl_token", token, q)
+}
+
+// UpdateAgentACLToken updates the agent's "agent" token. See updateToken
+// for more details
+func (a *Agent) UpdateAgentACLToken(token string, q *WriteOptions) (*WriteMeta, error) {
+	return a.updateTokenFallback("agent", "acl_agent_token", token, q)
+}
+
+// UpdateAgentMasterACLToken updates the agent's "agent_master" token. See updateToken
+// for more details
+func (a *Agent) UpdateAgentMasterACLToken(token string, q *WriteOptions) (*WriteMeta, error) {
+	return a.updateTokenFallback("agent_master", "acl_agent_master_token", token, q)
+}
+
+// UpdateReplicationACLToken updates the agent's "replication" token. See updateToken
+// for more details
+func (a *Agent) UpdateReplicationACLToken(token string, q *WriteOptions) (*WriteMeta, error) {
+	return a.updateTokenFallback("replication", "acl_replication_token", token, q)
+}
+
+// updateToken can be used to update one of an agent's ACL tokens after the agent has
+// started. The tokens are may not be persisted, so will need to be updated again if
+// the agent is restarted unless the agent is configured to persist them.
 func (a *Agent) updateToken(target, token string, q *WriteOptions) (*WriteMeta, error) {
+	meta, _, err := a.updateTokenOnce(target, token, q)
+	return meta, err
+}
+
+func (a *Agent) updateTokenFallback(target, fallback, token string, q *WriteOptions) (*WriteMeta, error) {
+	meta, status, err := a.updateTokenOnce(target, token, q)
+	if err != nil && status == 404 {
+		meta, _, err = a.updateTokenOnce(fallback, token, q)
+	}
+	return meta, err
+}
+
+func (a *Agent) updateTokenOnce(target, token string, q *WriteOptions) (*WriteMeta, int, error) {
 	r := a.c.newRequest("PUT", fmt.Sprintf("/v1/agent/token/%s", target))
 	r.setWriteOptions(q)
 	r.obj = &AgentToken{Token: token}
 	rtt, resp, err := requireOK(a.c.doRequest(r))
 	if err != nil {
-		return nil, err
+		return nil, resp.StatusCode, err
 	}
 	resp.Body.Close()
 
 	wm := &WriteMeta{RequestTime: rtt}
-	return wm, nil
+	return wm, resp.StatusCode, nil
 }

api/agent_test.go

@@ -1260,6 +1260,22 @@ func TestAPI_AgentUpdateToken(t *testing.T) {
 	if _, err := agent.UpdateACLReplicationToken("root", nil); err != nil {
 		t.Fatalf("err: %v", err)
 	}
+
+	if _, err := agent.UpdateDefaultACLToken("root", nil); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, err := agent.UpdateAgentACLToken("root", nil); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, err := agent.UpdateAgentMasterACLToken("root", nil); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, err := agent.UpdateReplicationACLToken("root", nil); err != nil {
+		t.Fatalf("err: %v", err)
+	}
 }
 
 func TestAPI_AgentConnectCARoots_empty(t *testing.T) {

command/acl/agenttokens/agent_tokens.go

@@ -51,13 +51,13 @@ func (c *cmd) Run(args []string) int {
 
 	switch tokenType {
 	case "default":
-		_, err = client.Agent().UpdateACLToken(token, nil)
+		_, err = client.Agent().UpdateDefaultACLToken(token, nil)
 	case "agent":
-		_, err = client.Agent().UpdateACLAgentToken(token, nil)
+		_, err = client.Agent().UpdateAgentACLToken(token, nil)
 	case "master":
-		_, err = client.Agent().UpdateACLAgentMasterToken(token, nil)
+		_, err = client.Agent().UpdateAgentMasterACLToken(token, nil)
 	case "replication":
-		_, err = client.Agent().UpdateACLReplicationToken(token, nil)
+		_, err = client.Agent().UpdateReplicationACLToken(token, nil)
 	default:
 		c.UI.Error(fmt.Sprintf("Unknown token type"))
 		return 1

lib/file/atomic.go

@@ -11,13 +11,18 @@ import (
 // WriteAtomic writes the given contents to a temporary file in the same
 // directory, does an fsync and then renames the file to its real path
 func WriteAtomic(path string, contents []byte) error {
+	return WriteAtomicWithPerms(path, contents, 0700)
+}
+
+func WriteAtomicWithPerms(path string, contents []byte, permissions os.FileMode) error {
+
 	uuid, err := uuid.GenerateUUID()
 	if err != nil {
 		return err
 	}
 	tempPath := fmt.Sprintf("%s-%s.tmp", path, uuid)
 
-	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+	if err := os.MkdirAll(filepath.Dir(path), permissions); err != nil {
 		return err
 	}
 	fh, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)

website/source/api/agent.html.md

@@ -516,8 +516,24 @@ $ curl \
 
 This endpoint updates the ACL tokens currently in use by the agent. It can be
 used to introduce ACL tokens to the agent for the first time, or to update
-tokens that were initially loaded from the agent's configuration. Tokens are
+tokens that were initially loaded from the agent's configuration. Tokens will be persisted
-not persisted, so will need to be updated again if the agent is restarted.
+only if the [`acl.enable_token_persistence`](/docs/agent/options.html#acl_enable_token_persistence)
+configuration is `true`. When not being persisted, they will need to be reset if the agent
+is restarted.
+
+| Method | Path                        | Produces                   |
+| ------ | --------------------------- | -------------------------- |
+| `PUT`  | `/agent/token/default`      | `application/json`         |
+| `PUT`  | `/agent/token/agent`        | `application/json`         |
+| `PUT`  | `/agent/token/agent_master` | `application/json`         |
+| `PUT`  | `/agent/token/replication`  | `application/json`         |
+
+The paths above correspond to the token names as found in the agent configuration:
+[`default`](/docs/agent/options.html#acl_tokens_default), [`agent`](/docs/agent/options.html#acl_tokens_agent),
+[`agent_master`](/docs/agent/options.html#acl_tokens_agent_master), and
+[`replication`](/docs/agent/options.html#acl_tokens_replication).
+
+-> **Deprecation Note:** The following paths were deprecated in version 1.4.3
 
 | Method | Path                                  | Produces                   |
 | ------ | ------------------------------------- | -------------------------- |
@@ -527,9 +543,9 @@ not persisted, so will need to be updated again if the agent is restarted.
 | `PUT`  | `/agent/token/acl_replication_token`  | `application/json`         |
 
 The paths above correspond to the token names as found in the agent configuration:
-[`acl_token`](/docs/agent/options.html#acl_token), [`acl_agent_token`](/docs/agent/options.html#acl_agent_token),
+[`acl_token`](/docs/agent/options.html#acl_token_legacy), [`acl_agent_token`](/docs/agent/options.html#acl_agent_token_legacy),
-[`acl_agent_master_token`](/docs/agent/options.html#acl_agent_master_token), and
+[`acl_agent_master_token`](/docs/agent/options.html#acl_agent_master_token_legacy), and
-[`acl_replication_token`](/docs/agent/options.html#acl_replication_token).
+[`acl_replication_token`](/docs/agent/options.html#acl_replication_token_legacy).
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),

website/source/docs/agent/options.html.md

@@ -554,10 +554,13 @@ default will automatically work with some tooling.
 
      * <a name="acl_enable_key_list"></a><a href="#acl_enable_key_list">`enable_key_list`</a> - Either "enabled" or "disabled", defaults to "disabled". When enabled, the `list` permission will be required on the prefix being recursively read from the KV store. Regardless of being enabled, the full set of KV entries under the prefix will be filtered to remove any entries that the request's ACL token does not grant at least read persmissions. This option is only available in Consul 1.0 and newer.
 
-     * <a name=`acl_enable_token_replication"></a><a href="#acl_enable_token_replication">`enable_token_replication`</a> - By
+     * <a name="acl_enable_token_replication"></a><a href="#acl_enable_token_replication">`enable_token_replication`</a> - By
      default secondary Consul datacenters will perform replication of only ACL policies. Setting this configuration will
      also enable ACL token replication.
 
+     * <a name="acl_enable_token_persistence"></a><a href="#acl_enable_token_persistence">`enable_token_persistence`</a> - Either
+    `true` or `false`. When `true` tokens set using the API will be persisted to disk and reloaded when an agent restarts.
+
      * <a name="acl_tokens"></a><a href="#acl_tokens">`tokens`</a> - This object holds
      all of the configured ACL tokens for the agents usage.