Refactor AutoConfig RPC to not have a direct dependency on the Server type

Instead it has an interface which can be mocked for better unit testing that is deterministic and not prone to flakiness.
This commit is contained in:
Matt Keeler 2020-07-07 15:39:04 -04:00
parent b5b9c8d953
commit d2e4869c7c
No known key found for this signature in database
GPG Key ID: 04DBAE1857E0081B
9 changed files with 456 additions and 193 deletions

View File

@ -23,7 +23,7 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// AutoConfigRequest is the data structure to be sent along with the
// Cluster.AutoConfig RPC
// AutoConfig.InitialConfiguration RPC
type AutoConfigRequest struct {
// Datacenter is the local datacenter name. This wont actually be set by clients
// but rather will be set by the servers to allow for forwarding to
@ -113,7 +113,7 @@ func (m *AutoConfigRequest) GetConsulToken() string {
return ""
}
// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
type AutoConfigResponse struct {
Config *config.Config `protobuf:"bytes,1,opt,name=Config,proto3" json:"Config,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`

View File

@ -7,7 +7,7 @@ option go_package = "github.com/hashicorp/consul/agent/agentpb";
import "agent/agentpb/config/config.proto";
// AutoConfigRequest is the data structure to be sent along with the
// Cluster.AutoConfig RPC
// AutoConfig.InitialConfiguration RPC
message AutoConfigRequest {
// Datacenter is the local datacenter name. This wont actually be set by clients
// but rather will be set by the servers to allow for forwarding to
@ -30,7 +30,7 @@ message AutoConfigRequest {
string ConsulToken = 6;
}
// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
message AutoConfigResponse {
config.Config Config = 1;
}

View File

@ -279,7 +279,7 @@ func (ac *AutoConfig) InitialConfiguration(ctx context.Context) (*config.Runtime
}
// introToken is responsible for determining the correct intro token to use
// when making the initial Cluster.AutoConfig RPC request.
// when making the initial AutoConfig.InitialConfiguration RPC request.
func (ac *AutoConfig) introToken() (string, error) {
conf := ac.config.AutoConfig
// without an intro token or intro token file we cannot do anything
@ -431,7 +431,7 @@ func (ac *AutoConfig) recordAutoConfigReply(reply *agentpb.AutoConfigResponse) e
}
// getInitialConfigurationOnce will perform full server to TCPAddr resolution and
// loop through each host trying to make the Cluster.AutoConfig RPC call. When
// loop through each host trying to make the AutoConfig.InitialConfiguration RPC call. When
// successful the bool return will be true and the err value will indicate whether we
// successfully recorded the auto config settings (persisted to disk and stored internally
// on the AutoConfig object)
@ -462,9 +462,9 @@ func (ac *AutoConfig) getInitialConfigurationOnce(ctx context.Context) (bool, er
return false, ctx.Err()
}
ac.logger.Debug("making Cluster.AutoConfig RPC", "addr", addr.String())
if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "Cluster.AutoConfig", &request, &reply); err != nil {
ac.logger.Error("AutoConfig RPC failed", "addr", addr.String(), "error", err)
ac.logger.Debug("making AutoConfig.InitialConfiguration RPC", "addr", addr.String())
if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "AutoConfig.InitialConfiguration", &request, &reply); err != nil {
ac.logger.Error("AutoConfig.InitialConfiguration RPC failed", "addr", addr.String(), "error", err)
continue
}

View File

@ -207,7 +207,7 @@ func TestInitialConfiguration_cancelled(t *testing.T) {
JWT: "blarg",
}
directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "Cluster.AutoConfig", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "AutoConfig.InitialConfiguration", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
ac, err := New(WithBuilderOpts(builderOpts), WithTLSConfigurator(&tlsutil.Configurator{}), WithDirectRPC(&directRPC))
require.NoError(t, err)
require.NotNil(t, ac)
@ -289,7 +289,7 @@ func TestInitialConfiguration_success(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(populateResponse)
@ -344,7 +344,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -352,7 +352,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 2), Port: 8398},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -360,7 +360,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 3), Port: 8399},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -368,7 +368,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Once()
directRPC.On(
@ -376,7 +376,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(populateResponse)

View File

@ -4,16 +4,12 @@ import (
"context"
"encoding/base64"
"fmt"
"net"
"time"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/agentpb"
"github.com/hashicorp/consul/agent/agentpb/config"
"github.com/hashicorp/consul/agent/consul/authmethod/ssoauth"
"github.com/hashicorp/consul/agent/metadata"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/lib/template"
"github.com/hashicorp/consul/tlsutil"
bexpr "github.com/hashicorp/go-bexpr"
@ -55,7 +51,6 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
"segment": req.Segment,
}
// TODO (autoconf) check for JWT reuse if configured to do so.
for _, raw := range a.claimAssertions {
// validate and fill any HIL
filled, err := template.InterpolateHIL(raw, varMap, true)
@ -84,87 +79,86 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
}, nil
}
// Cluster endpoint is used for cluster configuration operations
type Cluster struct {
srv *Server
type AutoConfigBackend interface {
GetConfig() *Config
CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error)
DatacenterJoinAddresses(segment string) ([]string, error)
TLSConfigurator() *tlsutil.Configurator
ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error)
}
// AutoConfig endpoint is used for cluster auto configuration operations
type AutoConfig struct {
backend AutoConfigBackend
authorizer AutoConfigAuthorizer
}
// getBackendConfig is a small wrapper around the backend's GetConfig to ensure
// that all helper functions have a non-nil configuration to operate on.
func (ac *AutoConfig) getBackendConfig() *Config {
if conf := ac.backend.GetConfig(); conf != nil {
return conf
}
return DefaultConfig()
}
// updateTLSCertificatesInConfig will ensure that the TLS settings regarding how an agent is
// made aware of its certificates are populated. This will only work if connect is enabled and
// in some cases only if auto_encrypt is enabled on the servers. This endpoint has the option
// to configure auto_encrypt or potentially in the future to generate the certificates inline.
func (c *Cluster) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
if c.srv.config.AutoEncryptAllowTLS {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: true}
} else {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: false}
}
func (ac *AutoConfig) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: ac.getBackendConfig().AutoEncryptAllowTLS}
return nil
}
// updateACLtokensInConfig will configure all of the agents ACL settings and will populate
// the configuration with an agent token usable for all default agent operations.
func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
backendConf := ac.getBackendConfig()
acl := &config.ACL{
Enabled: c.srv.config.ACLsEnabled,
PolicyTTL: c.srv.config.ACLPolicyTTL.String(),
RoleTTL: c.srv.config.ACLRoleTTL.String(),
TokenTTL: c.srv.config.ACLTokenTTL.String(),
DisabledTTL: c.srv.config.ACLDisabledTTL.String(),
DownPolicy: c.srv.config.ACLDownPolicy,
DefaultPolicy: c.srv.config.ACLDefaultPolicy,
EnableKeyListPolicy: c.srv.config.ACLEnableKeyListPolicy,
Enabled: backendConf.ACLsEnabled,
PolicyTTL: backendConf.ACLPolicyTTL.String(),
RoleTTL: backendConf.ACLRoleTTL.String(),
TokenTTL: backendConf.ACLTokenTTL.String(),
DisabledTTL: backendConf.ACLDisabledTTL.String(),
DownPolicy: backendConf.ACLDownPolicy,
DefaultPolicy: backendConf.ACLDefaultPolicy,
EnableKeyListPolicy: backendConf.ACLEnableKeyListPolicy,
}
// when ACLs are enabled we want to create a local token with a node identity
if c.srv.config.ACLsEnabled {
// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
// token create RPC to the servers in the primary DC.
if !c.srv.LocalTokensEnabled() {
return fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", c.srv.config.Datacenter)
}
// generate the accessor id
accessor, err := lib.GenerateUUID(c.srv.checkTokenUUID)
if err != nil {
return err
}
// generate the secret id
secret, err := lib.GenerateUUID(c.srv.checkTokenUUID)
if err != nil {
return err
}
// set up the token
token := structs.ACLToken{
AccessorID: accessor,
SecretID: secret,
if backendConf.ACLsEnabled {
// set up the token template - the ids and create
template := structs.ACLToken{
Description: fmt.Sprintf("Auto Config Token for Node %q", opts.NodeName),
CreateTime: time.Now(),
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: opts.NodeName,
Datacenter: c.srv.config.Datacenter,
Datacenter: backendConf.Datacenter,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
req := structs.ACLTokenBatchSetRequest{
Tokens: structs.ACLTokens{&token},
CAS: false,
token, err := ac.backend.CreateACLToken(&template)
if err != nil {
return fmt.Errorf("Failed to generate an ACL token for node %q - %w", opts.NodeName, err)
}
// perform the request to mint the new token
if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
return err
}
// req := structs.ACLTokenBatchSetRequest{
// Tokens: structs.ACLTokens{&token},
// CAS: false,
// }
acl.Tokens = &config.ACLTokens{Agent: secret}
// // perform the request to mint the new token
// if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
// return err
// }
acl.Tokens = &config.ACLTokens{Agent: token.SecretID}
}
conf.ACL = acl
@ -173,19 +167,19 @@ func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config
// updateJoinAddressesInConfig determines the correct gossip endpoints that clients should
// be connecting to for joining the cluster based on the segment given in the opts parameter.
func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
members, err := c.srv.LANSegmentMembers(opts.SegmentName)
func (ac *AutoConfig) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
joinAddrs, err := ac.backend.DatacenterJoinAddresses(opts.SegmentName)
if err != nil {
return err
}
var joinAddrs []string
for _, m := range members {
if ok, _ := metadata.IsConsulServer(m); ok {
serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
joinAddrs = append(joinAddrs, serfAddr.String())
}
}
// var joinAddrs []string
// for _, m := range members {
// if ok, _ := metadata.IsConsulServer(m); ok {
// serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
// joinAddrs = append(joinAddrs, serfAddr.String())
// }
// }
if conf.Gossip == nil {
conf.Gossip = &config.Gossip{}
@ -196,9 +190,9 @@ func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *conf
}
// updateGossipEncryptionInConfig will populate the gossip encryption configuration settings
func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
// Add gossip encryption settings if there is any key loaded
memberlistConfig := c.srv.config.SerfLANConfig.MemberlistConfig
memberlistConfig := ac.getBackendConfig().SerfLANConfig.MemberlistConfig
if lanKeyring := memberlistConfig.Keyring; lanKeyring != nil {
if conf.Gossip == nil {
conf.Gossip = &config.Gossip{}
@ -221,13 +215,20 @@ func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *conf
// updateTLSSettingsInConfig will populate the TLS configuration settings but will not
// populate leaf or ca certficiates.
func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
tlsConfigurator := ac.backend.TLSConfigurator()
if tlsConfigurator == nil {
// TLS is not enabled?
return nil
}
// add in TLS configuration
if conf.TLS == nil {
conf.TLS = &config.TLS{}
}
conf.TLS.VerifyServerHostname = c.srv.tlsConfigurator.VerifyServerHostname()
base := c.srv.tlsConfigurator.Base()
conf.TLS.VerifyServerHostname = tlsConfigurator.VerifyServerHostname()
base := tlsConfigurator.Base()
conf.TLS.VerifyOutgoing = base.VerifyOutgoing
conf.TLS.MinVersion = base.TLSMinVersion
conf.TLS.PreferServerCipherSuites = base.PreferServerCipherSuites
@ -239,61 +240,65 @@ func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Co
// baseConfig will populate the configuration with some base settings such as the
// datacenter names, node name etc.
func (c *Cluster) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
backendConf := ac.getBackendConfig()
if opts.NodeName == "" {
return fmt.Errorf("Cannot generate auto config response without a node name")
}
conf.Datacenter = c.srv.config.Datacenter
conf.PrimaryDatacenter = c.srv.config.PrimaryDatacenter
conf.Datacenter = backendConf.Datacenter
conf.PrimaryDatacenter = backendConf.PrimaryDatacenter
conf.NodeName = opts.NodeName
conf.SegmentName = opts.SegmentName
return nil
}
type autoConfigUpdater func(c *Cluster, opts AutoConfigOptions, conf *config.Config) error
type autoConfigUpdater func(c *AutoConfig, opts AutoConfigOptions, conf *config.Config) error
var (
// variable holding the list of config updating functions to execute when generating
// the auto config response. This will allow for more easily adding extra self-contained
// configurators here in the future.
autoConfigUpdaters []autoConfigUpdater = []autoConfigUpdater{
(*Cluster).baseConfig,
(*Cluster).updateJoinAddressesInConfig,
(*Cluster).updateGossipEncryptionInConfig,
(*Cluster).updateTLSSettingsInConfig,
(*Cluster).updateACLsInConfig,
(*Cluster).updateTLSCertificatesInConfig,
(*AutoConfig).baseConfig,
(*AutoConfig).updateJoinAddressesInConfig,
(*AutoConfig).updateGossipEncryptionInConfig,
(*AutoConfig).updateTLSSettingsInConfig,
(*AutoConfig).updateACLsInConfig,
(*AutoConfig).updateTLSCertificatesInConfig,
}
)
// AgentAutoConfig will authorize the incoming request and then generate the configuration
// to push down to the client
func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
func (ac *AutoConfig) InitialConfiguration(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
backendConf := ac.getBackendConfig()
// default the datacenter to our datacenter - agents do not have to specify this as they may not
// yet know the datacenter name they are going to be in.
if req.Datacenter == "" {
req.Datacenter = c.srv.config.Datacenter
req.Datacenter = backendConf.Datacenter
}
// TODO (autoconf) Is performing auto configuration over the WAN really a bad idea?
if req.Datacenter != c.srv.config.Datacenter {
if req.Datacenter != backendConf.Datacenter {
return fmt.Errorf("invalid datacenter %q - agent auto configuration cannot target a remote datacenter", req.Datacenter)
}
// forward to the leader
if done, err := c.srv.forward("Cluster.AutoConfig", req, req, resp); done {
if done, err := ac.backend.ForwardRPC("AutoConfig.InitialConfiguration", req, req, resp); done {
return err
}
// TODO (autoconf) maybe panic instead?
if c.authorizer == nil {
if ac.authorizer == nil {
return fmt.Errorf("No Auto Config authorizer is configured")
}
// authorize the request with the configured authorizer
opts, err := c.authorizer.Authorize(req)
opts, err := ac.authorizer.Authorize(req)
if err != nil {
return err
}
@ -302,7 +307,7 @@ func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoC
// update all the configurations
for _, configFn := range autoConfigUpdaters {
if err := configFn(c, opts, conf); err != nil {
if err := configFn(ac, opts, conf); err != nil {
return err
}
}

View File

@ -19,11 +19,49 @@ import (
"github.com/hashicorp/consul/tlsutil"
"github.com/hashicorp/memberlist"
msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gopkg.in/square/go-jose.v2/jwt"
)
type mockAutoConfigBackend struct {
mock.Mock
}
func (m *mockAutoConfigBackend) GetConfig() *Config {
ret := m.Called()
// this handles converting an untyped nil to a typed nil
cfg, _ := ret.Get(0).(*Config)
return cfg
}
func (m *mockAutoConfigBackend) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
ret := m.Called(template)
// this handles converting an untyped nil to a typed nil
token, _ := ret.Get(0).(*structs.ACLToken)
return token, ret.Error(1)
}
func (m *mockAutoConfigBackend) DatacenterJoinAddresses(segment string) ([]string, error) {
ret := m.Called(segment)
// this handles converting an untyped nil to a typed nil
addrs, _ := ret.Get(0).([]string)
return addrs, ret.Error(1)
}
func (m *mockAutoConfigBackend) TLSConfigurator() *tlsutil.Configurator {
ret := m.Called()
// this handles converting an untyped nil to a typed nil
cfg, _ := ret.Get(0).(*tlsutil.Configurator)
return cfg
}
func (m *mockAutoConfigBackend) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
ret := m.Called(method, info, args, reply)
return ret.Bool(0), ret.Error(1)
}
func testJWTStandardClaims() jwt.Claims {
now := time.Now()
@ -48,13 +86,13 @@ func signJWTWithStandardClaims(t *testing.T, privKey string, claims interface{})
return signJWT(t, privKey, testJWTStandardClaims(), claims)
}
// TestClusterAutoConfig is really an integration test of all the moving parts of the Cluster.AutoConfig RPC.
// TestAutoConfigInitialConfiguration is really an integration test of all the moving parts of the AutoConfig.InitialConfiguration RPC.
// Full testing of the individual parts will not be done in this test:
//
// * Any implementations of the AutoConfigAuthorizer interface (although these test do use the jwtAuthorizer)
// * Each of the individual config generation functions. These can be unit tested separately and many wont
// require a running test server.
func TestClusterAutoConfig(t *testing.T) {
// * Each of the individual config generation functions. These can be unit tested separately and should NOT
// require running test servers
func TestAutoConfigInitialConfiguration(t *testing.T) {
type testCase struct {
request agentpb.AutoConfigRequest
expected agentpb.AutoConfigResponse
@ -227,7 +265,7 @@ func TestClusterAutoConfig(t *testing.T) {
for testName, tcase := range cases {
t.Run(testName, func(t *testing.T) {
var reply agentpb.AutoConfigResponse
err := msgpackrpc.CallWithCodec(codec, "Cluster.AutoConfig", &tcase.request, &reply)
err := msgpackrpc.CallWithCodec(codec, "AutoConfig.InitialConfiguration", &tcase.request, &reply)
if tcase.err != "" {
testutil.RequireErrorContains(t, err, tcase.err)
} else {
@ -241,7 +279,7 @@ func TestClusterAutoConfig(t *testing.T) {
}
}
func TestClusterAutoConfig_baseConfig(t *testing.T) {
func TestAutoConfig_baseConfig(t *testing.T) {
type testCase struct {
serverConfig Config
opts AutoConfigOptions
@ -277,25 +315,28 @@ func TestClusterAutoConfig_baseConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: &tcase.serverConfig,
},
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.serverConfig).Once()
ac := AutoConfig{
backend: backend,
}
var actual config.Config
err := cluster.baseConfig(tcase.opts, &actual)
err := ac.baseConfig(tcase.opts, &actual)
if tcase.err == "" {
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
} else {
testutil.RequireErrorContains(t, err, tcase.err)
}
backend.AssertExpectations(t)
})
}
}
func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
func TestAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
_, _, cacert, err := testTLSCertificates("server.dc1.consul")
require.NoError(t, err)
@ -365,16 +406,19 @@ func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
configurator, err := tlsutil.NewConfigurator(tcase.tlsConfig, logger)
require.NoError(t, err)
cluster := &Cluster{
srv: &Server{
tlsConfigurator: configurator,
},
backend := &mockAutoConfigBackend{}
backend.On("TLSConfigurator").Return(configurator).Once()
ac := &AutoConfig{
backend: backend,
}
var actual config.Config
err = cluster.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
err = ac.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
@ -436,18 +480,22 @@ func TestAutoConfig_updateGossipEncryptionInConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: DefaultConfig(),
},
cfg := DefaultConfig()
cfg.SerfLANConfig.MemberlistConfig = &tcase.conf
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(cfg).Once()
ac := AutoConfig{
backend: backend,
}
cluster.srv.config.SerfLANConfig.MemberlistConfig = &tcase.conf
var actual config.Config
err := cluster.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
err := ac.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
@ -481,40 +529,53 @@ func TestAutoConfig_updateTLSCertificatesInConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: &tcase.serverConfig,
},
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.serverConfig).Once()
ac := AutoConfig{
backend: backend,
}
var actual config.Config
err := cluster.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
err := ac.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
func TestAutoConfig_updateACLsInConfig(t *testing.T) {
type testCase struct {
patch func(c *Config)
expected config.Config
verify func(t *testing.T, c *config.Config)
err string
config Config
expected config.Config
expectACLToken bool
err error
}
const (
tokenAccessor = "b98761aa-c0ee-445b-9b0c-f54b56b47778"
tokenSecret = "1c96448a-ab04-4caa-982a-e8b095a111e2"
)
testDC := "dc1"
cases := map[string]testCase{
"enabled": {
patch: func(c *Config) {
c.ACLsEnabled = true
c.ACLPolicyTTL = 7 * time.Second
c.ACLRoleTTL = 10 * time.Second
c.ACLTokenTTL = 12 * time.Second
c.ACLDisabledTTL = 31 * time.Second
c.ACLDefaultPolicy = "allow"
c.ACLDownPolicy = "deny"
c.ACLEnableKeyListPolicy = true
config: Config{
Datacenter: testDC,
PrimaryDatacenter: testDC,
ACLsEnabled: true,
ACLPolicyTTL: 7 * time.Second,
ACLRoleTTL: 10 * time.Second,
ACLTokenTTL: 12 * time.Second,
ACLDisabledTTL: 31 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "deny",
ACLEnableKeyListPolicy: true,
},
expectACLToken: true,
expected: config.Config{
ACL: &config.ACL{
Enabled: true,
@ -525,32 +586,26 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
DownPolicy: "deny",
DefaultPolicy: "allow",
EnableKeyListPolicy: true,
Tokens: &config.ACLTokens{Agent: "verified"},
Tokens: &config.ACLTokens{
Agent: tokenSecret,
},
},
},
verify: func(t *testing.T, c *config.Config) {
t.Helper()
// the agent token secret is non-deterministically generated
// So we want to validate that one was set and overwrite with
// a value that the expected configurate wants.
require.NotNil(t, c)
require.NotNil(t, c.ACL)
require.NotNil(t, c.ACL.Tokens)
require.NotEmpty(t, c.ACL.Tokens.Agent)
c.ACL.Tokens.Agent = "verified"
},
},
"disabled": {
patch: func(c *Config) {
c.ACLsEnabled = false
c.ACLPolicyTTL = 7 * time.Second
c.ACLRoleTTL = 10 * time.Second
c.ACLTokenTTL = 12 * time.Second
c.ACLDisabledTTL = 31 * time.Second
c.ACLDefaultPolicy = "allow"
c.ACLDownPolicy = "deny"
c.ACLEnableKeyListPolicy = true
config: Config{
Datacenter: testDC,
PrimaryDatacenter: testDC,
ACLsEnabled: false,
ACLPolicyTTL: 7 * time.Second,
ACLRoleTTL: 10 * time.Second,
ACLTokenTTL: 12 * time.Second,
ACLDisabledTTL: 31 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "deny",
ACLEnableKeyListPolicy: true,
},
expectACLToken: false,
expected: config.Config{
ACL: &config.ACL{
Enabled: false,
@ -565,53 +620,79 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
},
},
"local-tokens-disabled": {
patch: func(c *Config) {
c.PrimaryDatacenter = "somewhere else"
config: Config{
Datacenter: testDC,
PrimaryDatacenter: "somewhere-else",
ACLsEnabled: true,
},
err: "Agent Auto Configuration requires local token usage to be enabled in this datacenter",
expectACLToken: true,
err: fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter"),
},
}
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
_, s, _ := testACLServerWithConfig(t, tcase.patch, false)
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.config).Once()
waitForLeaderEstablishment(t, s)
expectedTemplate := &structs.ACLToken{
Description: `Auto Config Token for Node "something"`,
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "something",
Datacenter: testDC,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
cluster := Cluster{srv: s}
testToken := &structs.ACLToken{
AccessorID: tokenAccessor,
SecretID: tokenSecret,
Description: `Auto Config Token for Node "something"`,
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "something",
Datacenter: testDC,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
if tcase.expectACLToken {
backend.On("CreateACLToken", expectedTemplate).Return(testToken, tcase.err).Once()
}
ac := AutoConfig{backend: backend}
var actual config.Config
err := cluster.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
if tcase.err != "" {
testutil.RequireErrorContains(t, err, tcase.err)
err := ac.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
if tcase.err != nil {
testutil.RequireErrorContains(t, err, tcase.err.Error())
} else {
require.NoError(t, err)
if tcase.verify != nil {
tcase.verify(t, &actual)
}
require.Equal(t, tcase.expected, actual)
}
backend.AssertExpectations(t)
})
}
}
func TestAutoConfig_updateJoinAddressesInConfig(t *testing.T) {
conf := testClusterConfig{
Datacenter: "primary",
Servers: 3,
}
addrs := []string{"198.18.0.7:8300", "198.18.0.1:8300"}
backend := &mockAutoConfigBackend{}
backend.On("DatacenterJoinAddresses", "").Return(addrs, nil).Once()
nodes := newTestCluster(t, &conf)
cluster := Cluster{srv: nodes.Servers[0]}
ac := AutoConfig{backend: backend}
var actual config.Config
err := cluster.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
err := ac.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
var expected []string
for _, srv := range nodes.Servers {
expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
}
require.NotNil(t, actual.Gossip)
require.ElementsMatch(t, expected, actual.Gossip.RetryJoinLAN)
require.ElementsMatch(t, addrs, actual.Gossip.RetryJoinLAN)
backend.AssertExpectations(t)
}

View File

@ -876,7 +876,7 @@ func (s *Server) setupRPC() error {
authz = &disabledAuthorizer{}
}
// now register with the insecure RPC server
s.insecureRPCServer.Register(&Cluster{srv: s, authorizer: authz})
s.insecureRPCServer.Register(&AutoConfig{backend: s, authorizer: authz})
ln, err := net.ListenTCP("tcp", s.config.RPCAddr)
if err != nil {
@ -1436,6 +1436,89 @@ func (s *Server) intentionReplicationEnabled() bool {
return s.config.ConnectEnabled && s.config.Datacenter != s.config.PrimaryDatacenter
}
// CreateACLToken will create an ACL token from the given template
func (s *Server) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
// token create RPC to the servers in the primary DC.
if !s.LocalTokensEnabled() {
return nil, fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", s.config.Datacenter)
}
newToken := *template
// generate the accessor id
if newToken.AccessorID == "" {
accessor, err := lib.GenerateUUID(s.checkTokenUUID)
if err != nil {
return nil, err
}
newToken.AccessorID = accessor
}
// generate the secret id
if newToken.SecretID == "" {
secret, err := lib.GenerateUUID(s.checkTokenUUID)
if err != nil {
return nil, err
}
newToken.SecretID = secret
}
newToken.CreateTime = time.Now()
req := structs.ACLTokenBatchSetRequest{
Tokens: structs.ACLTokens{&newToken},
CAS: false,
}
// perform the request to mint the new token
if _, err := s.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
return nil, err
}
// return the full token definition from the FSM
_, token, err := s.fsm.State().ACLTokenGetByAccessor(nil, newToken.AccessorID, &newToken.EnterpriseMeta)
return token, err
}
// DatacenterJoinAddresses will return all the strings suitable for usage in
// retry join operations to connect to the the LAN or LAN segment gossip pool.
func (s *Server) DatacenterJoinAddresses(segment string) ([]string, error) {
members, err := s.LANSegmentMembers(segment)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve members for segment %s - %w", segment, err)
}
var joinAddrs []string
for _, m := range members {
if ok, _ := metadata.IsConsulServer(m); ok {
serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
joinAddrs = append(joinAddrs, serfAddr.String())
}
}
return joinAddrs, nil
}
// ForwardRPC is basically just renaming forward, in a future commit I am going to actually do the rename.
func (s *Server) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
return s.forward(method, info, args, reply)
}
// GetConfig will return the servers configuration - this is needed to satisfy
// interfaces: AutoConfigDelegate
func (s *Server) GetConfig() *Config {
return s.config
}
// TLSConfigurator just returns the servers TLS Configurator - this is needed to satisfy
// interfaces: AutoConfigDelegate
func (s *Server) TLSConfigurator() *tlsutil.Configurator {
return s.tlsConfigurator
}
// peersInfoContent is used to help operators understand what happened to the
// peers.json file. This is written to a file called peers.info in the same
// location.

View File

@ -1534,3 +1534,97 @@ func TestServer_CALogging(t *testing.T) {
require.Contains(t, buf.String(), "consul CA provider configured")
}
func TestServer_DatacenterJoinAddresses(t *testing.T) {
conf := testClusterConfig{
Datacenter: "primary",
Servers: 3,
}
nodes := newTestCluster(t, &conf)
var expected []string
for _, srv := range nodes.Servers {
expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
}
actual, err := nodes.Servers[0].DatacenterJoinAddresses("")
require.NoError(t, err)
require.ElementsMatch(t, expected, actual)
}
func TestServer_CreateACLToken(t *testing.T) {
_, srv, codec := testACLServerWithConfig(t, nil, false)
waitForLeaderEstablishment(t, srv)
r1, err := upsertTestRole(codec, TestDefaultMasterToken, "dc1")
require.NoError(t, err)
t.Run("predefined-ids", func(t *testing.T) {
accessor := "554cd3ab-5d4e-4d6e-952e-4e8b6c77bfb3"
secret := "ef453f31-ad58-4ec8-8bf8-342e99763026"
in := &structs.ACLToken{
AccessorID: accessor,
SecretID: secret,
Description: "test",
Policies: []structs.ACLTokenPolicyLink{
{
ID: structs.ACLPolicyGlobalManagementID,
},
},
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "foo",
Datacenter: "bar",
},
},
ServiceIdentities: []*structs.ACLServiceIdentity{
{
ServiceName: "web",
},
},
Roles: []structs.ACLTokenRoleLink{
{
ID: r1.ID,
},
},
}
out, err := srv.CreateACLToken(in)
require.NoError(t, err)
require.Equal(t, accessor, out.AccessorID)
require.Equal(t, secret, out.SecretID)
require.Equal(t, "test", out.Description)
require.NotZero(t, out.CreateTime)
require.Len(t, out.Policies, 1)
require.Len(t, out.Roles, 1)
require.Len(t, out.NodeIdentities, 1)
require.Len(t, out.ServiceIdentities, 1)
require.Equal(t, structs.ACLPolicyGlobalManagementID, out.Policies[0].ID)
require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
require.Equal(t, "web", out.ServiceIdentities[0].ServiceName)
require.Equal(t, r1.ID, out.Roles[0].ID)
})
t.Run("autogen-ids", func(t *testing.T) {
in := &structs.ACLToken{
Description: "test",
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "foo",
Datacenter: "bar",
},
},
}
out, err := srv.CreateACLToken(in)
require.NoError(t, err)
require.NotEmpty(t, out.AccessorID)
require.NotEmpty(t, out.SecretID)
require.Equal(t, "test", out.Description)
require.NotZero(t, out.CreateTime)
require.Len(t, out.NodeIdentities, 1)
require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
})
}

View File

@ -553,7 +553,7 @@ func (p *ConnPool) RPC(
// secure or insecure variant depending on whether its an ongoing
// or first time config request. For now though this is fine until
// those ongoing requests are implemented.
if method == "AutoEncrypt.Sign" || method == "Cluster.AutoConfig" {
if method == "AutoEncrypt.Sign" || method == "AutoConfig.InitialConfiguration" {
return p.rpcInsecure(dc, addr, method, args, reply)
} else {
return p.rpc(dc, nodeName, addr, method, args, reply)