Refactor AutoConfig RPC to not have a direct dependency on the Server type

Instead it has an interface which can be mocked for better unit testing that is deterministic and not prone to flakiness.

# Conflicts:
#	agent/pool/pool.go
This commit is contained in:
Matt Keeler 2020-07-07 15:39:04 -04:00
parent af20cb1f99
commit 977eb725a7
No known key found for this signature in database
GPG Key ID: 04DBAE1857E0081B
9 changed files with 456 additions and 193 deletions

View File

@ -23,7 +23,7 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// AutoConfigRequest is the data structure to be sent along with the
// Cluster.AutoConfig RPC
// AutoConfig.InitialConfiguration RPC
type AutoConfigRequest struct {
// Datacenter is the local datacenter name. This wont actually be set by clients
// but rather will be set by the servers to allow for forwarding to
@ -113,7 +113,7 @@ func (m *AutoConfigRequest) GetConsulToken() string {
return ""
}
// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
type AutoConfigResponse struct {
Config *config.Config `protobuf:"bytes,1,opt,name=Config,proto3" json:"Config,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`

View File

@ -7,7 +7,7 @@ option go_package = "github.com/hashicorp/consul/agent/agentpb";
import "agent/agentpb/config/config.proto";
// AutoConfigRequest is the data structure to be sent along with the
// Cluster.AutoConfig RPC
// AutoConfig.InitialConfiguration RPC
message AutoConfigRequest {
// Datacenter is the local datacenter name. This wont actually be set by clients
// but rather will be set by the servers to allow for forwarding to
@ -30,7 +30,7 @@ message AutoConfigRequest {
string ConsulToken = 6;
}
// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
message AutoConfigResponse {
config.Config Config = 1;
}

View File

@ -279,7 +279,7 @@ func (ac *AutoConfig) InitialConfiguration(ctx context.Context) (*config.Runtime
}
// introToken is responsible for determining the correct intro token to use
// when making the initial Cluster.AutoConfig RPC request.
// when making the initial AutoConfig.InitialConfiguration RPC request.
func (ac *AutoConfig) introToken() (string, error) {
conf := ac.config.AutoConfig
// without an intro token or intro token file we cannot do anything
@ -431,7 +431,7 @@ func (ac *AutoConfig) recordAutoConfigReply(reply *agentpb.AutoConfigResponse) e
}
// getInitialConfigurationOnce will perform full server to TCPAddr resolution and
// loop through each host trying to make the Cluster.AutoConfig RPC call. When
// loop through each host trying to make the AutoConfig.InitialConfiguration RPC call. When
// successful the bool return will be true and the err value will indicate whether we
// successfully recorded the auto config settings (persisted to disk and stored internally
// on the AutoConfig object)
@ -462,9 +462,9 @@ func (ac *AutoConfig) getInitialConfigurationOnce(ctx context.Context) (bool, er
return false, ctx.Err()
}
ac.logger.Debug("making Cluster.AutoConfig RPC", "addr", addr.String())
if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "Cluster.AutoConfig", &request, &reply); err != nil {
ac.logger.Error("AutoConfig RPC failed", "addr", addr.String(), "error", err)
ac.logger.Debug("making AutoConfig.InitialConfiguration RPC", "addr", addr.String())
if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "AutoConfig.InitialConfiguration", &request, &reply); err != nil {
ac.logger.Error("AutoConfig.InitialConfiguration RPC failed", "addr", addr.String(), "error", err)
continue
}

View File

@ -207,7 +207,7 @@ func TestInitialConfiguration_cancelled(t *testing.T) {
JWT: "blarg",
}
directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "Cluster.AutoConfig", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "AutoConfig.InitialConfiguration", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
ac, err := New(WithBuilderOpts(builderOpts), WithTLSConfigurator(&tlsutil.Configurator{}), WithDirectRPC(&directRPC))
require.NoError(t, err)
require.NotNil(t, ac)
@ -289,7 +289,7 @@ func TestInitialConfiguration_success(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(populateResponse)
@ -344,7 +344,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -352,7 +352,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 2), Port: 8398},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -360,7 +360,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 3), Port: 8399},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
directRPC.On(
@ -368,7 +368,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Once()
directRPC.On(
@ -376,7 +376,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
"dc1",
"autoconf",
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
"Cluster.AutoConfig",
"AutoConfig.InitialConfiguration",
&expectedRequest,
&agentpb.AutoConfigResponse{}).Return(populateResponse)

View File

@ -4,16 +4,12 @@ import (
"context"
"encoding/base64"
"fmt"
"net"
"time"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/agentpb"
"github.com/hashicorp/consul/agent/agentpb/config"
"github.com/hashicorp/consul/agent/consul/authmethod/ssoauth"
"github.com/hashicorp/consul/agent/metadata"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/lib/template"
"github.com/hashicorp/consul/tlsutil"
bexpr "github.com/hashicorp/go-bexpr"
@ -55,7 +51,6 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
"segment": req.Segment,
}
// TODO (autoconf) check for JWT reuse if configured to do so.
for _, raw := range a.claimAssertions {
// validate and fill any HIL
filled, err := template.InterpolateHIL(raw, varMap, true)
@ -84,87 +79,86 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
}, nil
}
// Cluster endpoint is used for cluster configuration operations
type Cluster struct {
srv *Server
type AutoConfigBackend interface {
GetConfig() *Config
CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error)
DatacenterJoinAddresses(segment string) ([]string, error)
TLSConfigurator() *tlsutil.Configurator
ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error)
}
// AutoConfig endpoint is used for cluster auto configuration operations
type AutoConfig struct {
backend AutoConfigBackend
authorizer AutoConfigAuthorizer
}
// getBackendConfig is a small wrapper around the backend's GetConfig to ensure
// that all helper functions have a non-nil configuration to operate on.
func (ac *AutoConfig) getBackendConfig() *Config {
if conf := ac.backend.GetConfig(); conf != nil {
return conf
}
return DefaultConfig()
}
// updateTLSCertificatesInConfig will ensure that the TLS settings regarding how an agent is
// made aware of its certificates are populated. This will only work if connect is enabled and
// in some cases only if auto_encrypt is enabled on the servers. This endpoint has the option
// to configure auto_encrypt or potentially in the future to generate the certificates inline.
func (c *Cluster) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
if c.srv.config.AutoEncryptAllowTLS {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: true}
} else {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: false}
}
func (ac *AutoConfig) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
conf.AutoEncrypt = &config.AutoEncrypt{TLS: ac.getBackendConfig().AutoEncryptAllowTLS}
return nil
}
// updateACLtokensInConfig will configure all of the agents ACL settings and will populate
// the configuration with an agent token usable for all default agent operations.
func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
backendConf := ac.getBackendConfig()
acl := &config.ACL{
Enabled: c.srv.config.ACLsEnabled,
PolicyTTL: c.srv.config.ACLPolicyTTL.String(),
RoleTTL: c.srv.config.ACLRoleTTL.String(),
TokenTTL: c.srv.config.ACLTokenTTL.String(),
DisabledTTL: c.srv.config.ACLDisabledTTL.String(),
DownPolicy: c.srv.config.ACLDownPolicy,
DefaultPolicy: c.srv.config.ACLDefaultPolicy,
EnableKeyListPolicy: c.srv.config.ACLEnableKeyListPolicy,
Enabled: backendConf.ACLsEnabled,
PolicyTTL: backendConf.ACLPolicyTTL.String(),
RoleTTL: backendConf.ACLRoleTTL.String(),
TokenTTL: backendConf.ACLTokenTTL.String(),
DisabledTTL: backendConf.ACLDisabledTTL.String(),
DownPolicy: backendConf.ACLDownPolicy,
DefaultPolicy: backendConf.ACLDefaultPolicy,
EnableKeyListPolicy: backendConf.ACLEnableKeyListPolicy,
}
// when ACLs are enabled we want to create a local token with a node identity
if c.srv.config.ACLsEnabled {
// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
// token create RPC to the servers in the primary DC.
if !c.srv.LocalTokensEnabled() {
return fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", c.srv.config.Datacenter)
}
// generate the accessor id
accessor, err := lib.GenerateUUID(c.srv.checkTokenUUID)
if err != nil {
return err
}
// generate the secret id
secret, err := lib.GenerateUUID(c.srv.checkTokenUUID)
if err != nil {
return err
}
// set up the token
token := structs.ACLToken{
AccessorID: accessor,
SecretID: secret,
if backendConf.ACLsEnabled {
// set up the token template - the ids and create
template := structs.ACLToken{
Description: fmt.Sprintf("Auto Config Token for Node %q", opts.NodeName),
CreateTime: time.Now(),
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: opts.NodeName,
Datacenter: c.srv.config.Datacenter,
Datacenter: backendConf.Datacenter,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
req := structs.ACLTokenBatchSetRequest{
Tokens: structs.ACLTokens{&token},
CAS: false,
token, err := ac.backend.CreateACLToken(&template)
if err != nil {
return fmt.Errorf("Failed to generate an ACL token for node %q - %w", opts.NodeName, err)
}
// perform the request to mint the new token
if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
return err
}
// req := structs.ACLTokenBatchSetRequest{
// Tokens: structs.ACLTokens{&token},
// CAS: false,
// }
acl.Tokens = &config.ACLTokens{Agent: secret}
// // perform the request to mint the new token
// if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
// return err
// }
acl.Tokens = &config.ACLTokens{Agent: token.SecretID}
}
conf.ACL = acl
@ -173,19 +167,19 @@ func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config
// updateJoinAddressesInConfig determines the correct gossip endpoints that clients should
// be connecting to for joining the cluster based on the segment given in the opts parameter.
func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
members, err := c.srv.LANSegmentMembers(opts.SegmentName)
func (ac *AutoConfig) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
joinAddrs, err := ac.backend.DatacenterJoinAddresses(opts.SegmentName)
if err != nil {
return err
}
var joinAddrs []string
for _, m := range members {
if ok, _ := metadata.IsConsulServer(m); ok {
serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
joinAddrs = append(joinAddrs, serfAddr.String())
}
}
// var joinAddrs []string
// for _, m := range members {
// if ok, _ := metadata.IsConsulServer(m); ok {
// serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
// joinAddrs = append(joinAddrs, serfAddr.String())
// }
// }
if conf.Gossip == nil {
conf.Gossip = &config.Gossip{}
@ -196,9 +190,9 @@ func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *conf
}
// updateGossipEncryptionInConfig will populate the gossip encryption configuration settings
func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
// Add gossip encryption settings if there is any key loaded
memberlistConfig := c.srv.config.SerfLANConfig.MemberlistConfig
memberlistConfig := ac.getBackendConfig().SerfLANConfig.MemberlistConfig
if lanKeyring := memberlistConfig.Keyring; lanKeyring != nil {
if conf.Gossip == nil {
conf.Gossip = &config.Gossip{}
@ -221,13 +215,20 @@ func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *conf
// updateTLSSettingsInConfig will populate the TLS configuration settings but will not
// populate leaf or ca certficiates.
func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
tlsConfigurator := ac.backend.TLSConfigurator()
if tlsConfigurator == nil {
// TLS is not enabled?
return nil
}
// add in TLS configuration
if conf.TLS == nil {
conf.TLS = &config.TLS{}
}
conf.TLS.VerifyServerHostname = c.srv.tlsConfigurator.VerifyServerHostname()
base := c.srv.tlsConfigurator.Base()
conf.TLS.VerifyServerHostname = tlsConfigurator.VerifyServerHostname()
base := tlsConfigurator.Base()
conf.TLS.VerifyOutgoing = base.VerifyOutgoing
conf.TLS.MinVersion = base.TLSMinVersion
conf.TLS.PreferServerCipherSuites = base.PreferServerCipherSuites
@ -239,61 +240,65 @@ func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Co
// baseConfig will populate the configuration with some base settings such as the
// datacenter names, node name etc.
func (c *Cluster) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
func (ac *AutoConfig) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
backendConf := ac.getBackendConfig()
if opts.NodeName == "" {
return fmt.Errorf("Cannot generate auto config response without a node name")
}
conf.Datacenter = c.srv.config.Datacenter
conf.PrimaryDatacenter = c.srv.config.PrimaryDatacenter
conf.Datacenter = backendConf.Datacenter
conf.PrimaryDatacenter = backendConf.PrimaryDatacenter
conf.NodeName = opts.NodeName
conf.SegmentName = opts.SegmentName
return nil
}
type autoConfigUpdater func(c *Cluster, opts AutoConfigOptions, conf *config.Config) error
type autoConfigUpdater func(c *AutoConfig, opts AutoConfigOptions, conf *config.Config) error
var (
// variable holding the list of config updating functions to execute when generating
// the auto config response. This will allow for more easily adding extra self-contained
// configurators here in the future.
autoConfigUpdaters []autoConfigUpdater = []autoConfigUpdater{
(*Cluster).baseConfig,
(*Cluster).updateJoinAddressesInConfig,
(*Cluster).updateGossipEncryptionInConfig,
(*Cluster).updateTLSSettingsInConfig,
(*Cluster).updateACLsInConfig,
(*Cluster).updateTLSCertificatesInConfig,
(*AutoConfig).baseConfig,
(*AutoConfig).updateJoinAddressesInConfig,
(*AutoConfig).updateGossipEncryptionInConfig,
(*AutoConfig).updateTLSSettingsInConfig,
(*AutoConfig).updateACLsInConfig,
(*AutoConfig).updateTLSCertificatesInConfig,
}
)
// AgentAutoConfig will authorize the incoming request and then generate the configuration
// to push down to the client
func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
func (ac *AutoConfig) InitialConfiguration(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
backendConf := ac.getBackendConfig()
// default the datacenter to our datacenter - agents do not have to specify this as they may not
// yet know the datacenter name they are going to be in.
if req.Datacenter == "" {
req.Datacenter = c.srv.config.Datacenter
req.Datacenter = backendConf.Datacenter
}
// TODO (autoconf) Is performing auto configuration over the WAN really a bad idea?
if req.Datacenter != c.srv.config.Datacenter {
if req.Datacenter != backendConf.Datacenter {
return fmt.Errorf("invalid datacenter %q - agent auto configuration cannot target a remote datacenter", req.Datacenter)
}
// forward to the leader
if done, err := c.srv.forward("Cluster.AutoConfig", req, req, resp); done {
if done, err := ac.backend.ForwardRPC("AutoConfig.InitialConfiguration", req, req, resp); done {
return err
}
// TODO (autoconf) maybe panic instead?
if c.authorizer == nil {
if ac.authorizer == nil {
return fmt.Errorf("No Auto Config authorizer is configured")
}
// authorize the request with the configured authorizer
opts, err := c.authorizer.Authorize(req)
opts, err := ac.authorizer.Authorize(req)
if err != nil {
return err
}
@ -302,7 +307,7 @@ func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoC
// update all the configurations
for _, configFn := range autoConfigUpdaters {
if err := configFn(c, opts, conf); err != nil {
if err := configFn(ac, opts, conf); err != nil {
return err
}
}

View File

@ -19,11 +19,49 @@ import (
"github.com/hashicorp/consul/tlsutil"
"github.com/hashicorp/memberlist"
msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gopkg.in/square/go-jose.v2/jwt"
)
type mockAutoConfigBackend struct {
mock.Mock
}
func (m *mockAutoConfigBackend) GetConfig() *Config {
ret := m.Called()
// this handles converting an untyped nil to a typed nil
cfg, _ := ret.Get(0).(*Config)
return cfg
}
func (m *mockAutoConfigBackend) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
ret := m.Called(template)
// this handles converting an untyped nil to a typed nil
token, _ := ret.Get(0).(*structs.ACLToken)
return token, ret.Error(1)
}
func (m *mockAutoConfigBackend) DatacenterJoinAddresses(segment string) ([]string, error) {
ret := m.Called(segment)
// this handles converting an untyped nil to a typed nil
addrs, _ := ret.Get(0).([]string)
return addrs, ret.Error(1)
}
func (m *mockAutoConfigBackend) TLSConfigurator() *tlsutil.Configurator {
ret := m.Called()
// this handles converting an untyped nil to a typed nil
cfg, _ := ret.Get(0).(*tlsutil.Configurator)
return cfg
}
func (m *mockAutoConfigBackend) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
ret := m.Called(method, info, args, reply)
return ret.Bool(0), ret.Error(1)
}
func testJWTStandardClaims() jwt.Claims {
now := time.Now()
@ -48,13 +86,13 @@ func signJWTWithStandardClaims(t *testing.T, privKey string, claims interface{})
return signJWT(t, privKey, testJWTStandardClaims(), claims)
}
// TestClusterAutoConfig is really an integration test of all the moving parts of the Cluster.AutoConfig RPC.
// TestAutoConfigInitialConfiguration is really an integration test of all the moving parts of the AutoConfig.InitialConfiguration RPC.
// Full testing of the individual parts will not be done in this test:
//
// * Any implementations of the AutoConfigAuthorizer interface (although these test do use the jwtAuthorizer)
// * Each of the individual config generation functions. These can be unit tested separately and many wont
// require a running test server.
func TestClusterAutoConfig(t *testing.T) {
// * Each of the individual config generation functions. These can be unit tested separately and should NOT
// require running test servers
func TestAutoConfigInitialConfiguration(t *testing.T) {
type testCase struct {
request agentpb.AutoConfigRequest
expected agentpb.AutoConfigResponse
@ -227,7 +265,7 @@ func TestClusterAutoConfig(t *testing.T) {
for testName, tcase := range cases {
t.Run(testName, func(t *testing.T) {
var reply agentpb.AutoConfigResponse
err := msgpackrpc.CallWithCodec(codec, "Cluster.AutoConfig", &tcase.request, &reply)
err := msgpackrpc.CallWithCodec(codec, "AutoConfig.InitialConfiguration", &tcase.request, &reply)
if tcase.err != "" {
testutil.RequireErrorContains(t, err, tcase.err)
} else {
@ -241,7 +279,7 @@ func TestClusterAutoConfig(t *testing.T) {
}
}
func TestClusterAutoConfig_baseConfig(t *testing.T) {
func TestAutoConfig_baseConfig(t *testing.T) {
type testCase struct {
serverConfig Config
opts AutoConfigOptions
@ -277,25 +315,28 @@ func TestClusterAutoConfig_baseConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: &tcase.serverConfig,
},
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.serverConfig).Once()
ac := AutoConfig{
backend: backend,
}
var actual config.Config
err := cluster.baseConfig(tcase.opts, &actual)
err := ac.baseConfig(tcase.opts, &actual)
if tcase.err == "" {
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
} else {
testutil.RequireErrorContains(t, err, tcase.err)
}
backend.AssertExpectations(t)
})
}
}
func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
func TestAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
_, _, cacert, err := testTLSCertificates("server.dc1.consul")
require.NoError(t, err)
@ -365,16 +406,19 @@ func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
configurator, err := tlsutil.NewConfigurator(tcase.tlsConfig, logger)
require.NoError(t, err)
cluster := &Cluster{
srv: &Server{
tlsConfigurator: configurator,
},
backend := &mockAutoConfigBackend{}
backend.On("TLSConfigurator").Return(configurator).Once()
ac := &AutoConfig{
backend: backend,
}
var actual config.Config
err = cluster.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
err = ac.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
@ -436,18 +480,22 @@ func TestAutoConfig_updateGossipEncryptionInConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: DefaultConfig(),
},
cfg := DefaultConfig()
cfg.SerfLANConfig.MemberlistConfig = &tcase.conf
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(cfg).Once()
ac := AutoConfig{
backend: backend,
}
cluster.srv.config.SerfLANConfig.MemberlistConfig = &tcase.conf
var actual config.Config
err := cluster.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
err := ac.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
@ -481,40 +529,53 @@ func TestAutoConfig_updateTLSCertificatesInConfig(t *testing.T) {
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
cluster := Cluster{
srv: &Server{
config: &tcase.serverConfig,
},
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.serverConfig).Once()
ac := AutoConfig{
backend: backend,
}
var actual config.Config
err := cluster.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
err := ac.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
require.Equal(t, tcase.expected, actual)
backend.AssertExpectations(t)
})
}
}
func TestAutoConfig_updateACLsInConfig(t *testing.T) {
type testCase struct {
patch func(c *Config)
expected config.Config
verify func(t *testing.T, c *config.Config)
err string
config Config
expected config.Config
expectACLToken bool
err error
}
const (
tokenAccessor = "b98761aa-c0ee-445b-9b0c-f54b56b47778"
tokenSecret = "1c96448a-ab04-4caa-982a-e8b095a111e2"
)
testDC := "dc1"
cases := map[string]testCase{
"enabled": {
patch: func(c *Config) {
c.ACLsEnabled = true
c.ACLPolicyTTL = 7 * time.Second
c.ACLRoleTTL = 10 * time.Second
c.ACLTokenTTL = 12 * time.Second
c.ACLDisabledTTL = 31 * time.Second
c.ACLDefaultPolicy = "allow"
c.ACLDownPolicy = "deny"
c.ACLEnableKeyListPolicy = true
config: Config{
Datacenter: testDC,
PrimaryDatacenter: testDC,
ACLsEnabled: true,
ACLPolicyTTL: 7 * time.Second,
ACLRoleTTL: 10 * time.Second,
ACLTokenTTL: 12 * time.Second,
ACLDisabledTTL: 31 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "deny",
ACLEnableKeyListPolicy: true,
},
expectACLToken: true,
expected: config.Config{
ACL: &config.ACL{
Enabled: true,
@ -525,32 +586,26 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
DownPolicy: "deny",
DefaultPolicy: "allow",
EnableKeyListPolicy: true,
Tokens: &config.ACLTokens{Agent: "verified"},
Tokens: &config.ACLTokens{
Agent: tokenSecret,
},
},
},
verify: func(t *testing.T, c *config.Config) {
t.Helper()
// the agent token secret is non-deterministically generated
// So we want to validate that one was set and overwrite with
// a value that the expected configurate wants.
require.NotNil(t, c)
require.NotNil(t, c.ACL)
require.NotNil(t, c.ACL.Tokens)
require.NotEmpty(t, c.ACL.Tokens.Agent)
c.ACL.Tokens.Agent = "verified"
},
},
"disabled": {
patch: func(c *Config) {
c.ACLsEnabled = false
c.ACLPolicyTTL = 7 * time.Second
c.ACLRoleTTL = 10 * time.Second
c.ACLTokenTTL = 12 * time.Second
c.ACLDisabledTTL = 31 * time.Second
c.ACLDefaultPolicy = "allow"
c.ACLDownPolicy = "deny"
c.ACLEnableKeyListPolicy = true
config: Config{
Datacenter: testDC,
PrimaryDatacenter: testDC,
ACLsEnabled: false,
ACLPolicyTTL: 7 * time.Second,
ACLRoleTTL: 10 * time.Second,
ACLTokenTTL: 12 * time.Second,
ACLDisabledTTL: 31 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "deny",
ACLEnableKeyListPolicy: true,
},
expectACLToken: false,
expected: config.Config{
ACL: &config.ACL{
Enabled: false,
@ -565,53 +620,79 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
},
},
"local-tokens-disabled": {
patch: func(c *Config) {
c.PrimaryDatacenter = "somewhere else"
config: Config{
Datacenter: testDC,
PrimaryDatacenter: "somewhere-else",
ACLsEnabled: true,
},
err: "Agent Auto Configuration requires local token usage to be enabled in this datacenter",
expectACLToken: true,
err: fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter"),
},
}
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
_, s, _ := testACLServerWithConfig(t, tcase.patch, false)
backend := &mockAutoConfigBackend{}
backend.On("GetConfig").Return(&tcase.config).Once()
waitForLeaderEstablishment(t, s)
expectedTemplate := &structs.ACLToken{
Description: `Auto Config Token for Node "something"`,
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "something",
Datacenter: testDC,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
cluster := Cluster{srv: s}
testToken := &structs.ACLToken{
AccessorID: tokenAccessor,
SecretID: tokenSecret,
Description: `Auto Config Token for Node "something"`,
Local: true,
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "something",
Datacenter: testDC,
},
},
EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
}
if tcase.expectACLToken {
backend.On("CreateACLToken", expectedTemplate).Return(testToken, tcase.err).Once()
}
ac := AutoConfig{backend: backend}
var actual config.Config
err := cluster.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
if tcase.err != "" {
testutil.RequireErrorContains(t, err, tcase.err)
err := ac.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
if tcase.err != nil {
testutil.RequireErrorContains(t, err, tcase.err.Error())
} else {
require.NoError(t, err)
if tcase.verify != nil {
tcase.verify(t, &actual)
}
require.Equal(t, tcase.expected, actual)
}
backend.AssertExpectations(t)
})
}
}
func TestAutoConfig_updateJoinAddressesInConfig(t *testing.T) {
conf := testClusterConfig{
Datacenter: "primary",
Servers: 3,
}
addrs := []string{"198.18.0.7:8300", "198.18.0.1:8300"}
backend := &mockAutoConfigBackend{}
backend.On("DatacenterJoinAddresses", "").Return(addrs, nil).Once()
nodes := newTestCluster(t, &conf)
cluster := Cluster{srv: nodes.Servers[0]}
ac := AutoConfig{backend: backend}
var actual config.Config
err := cluster.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
err := ac.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
require.NoError(t, err)
var expected []string
for _, srv := range nodes.Servers {
expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
}
require.NotNil(t, actual.Gossip)
require.ElementsMatch(t, expected, actual.Gossip.RetryJoinLAN)
require.ElementsMatch(t, addrs, actual.Gossip.RetryJoinLAN)
backend.AssertExpectations(t)
}

View File

@ -876,7 +876,7 @@ func (s *Server) setupRPC() error {
authz = &disabledAuthorizer{}
}
// now register with the insecure RPC server
s.insecureRPCServer.Register(&Cluster{srv: s, authorizer: authz})
s.insecureRPCServer.Register(&AutoConfig{backend: s, authorizer: authz})
ln, err := net.ListenTCP("tcp", s.config.RPCAddr)
if err != nil {
@ -1436,6 +1436,89 @@ func (s *Server) intentionReplicationEnabled() bool {
return s.config.ConnectEnabled && s.config.Datacenter != s.config.PrimaryDatacenter
}
// CreateACLToken will create an ACL token from the given template
func (s *Server) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
// token create RPC to the servers in the primary DC.
if !s.LocalTokensEnabled() {
return nil, fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", s.config.Datacenter)
}
newToken := *template
// generate the accessor id
if newToken.AccessorID == "" {
accessor, err := lib.GenerateUUID(s.checkTokenUUID)
if err != nil {
return nil, err
}
newToken.AccessorID = accessor
}
// generate the secret id
if newToken.SecretID == "" {
secret, err := lib.GenerateUUID(s.checkTokenUUID)
if err != nil {
return nil, err
}
newToken.SecretID = secret
}
newToken.CreateTime = time.Now()
req := structs.ACLTokenBatchSetRequest{
Tokens: structs.ACLTokens{&newToken},
CAS: false,
}
// perform the request to mint the new token
if _, err := s.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
return nil, err
}
// return the full token definition from the FSM
_, token, err := s.fsm.State().ACLTokenGetByAccessor(nil, newToken.AccessorID, &newToken.EnterpriseMeta)
return token, err
}
// DatacenterJoinAddresses will return all the strings suitable for usage in
// retry join operations to connect to the the LAN or LAN segment gossip pool.
func (s *Server) DatacenterJoinAddresses(segment string) ([]string, error) {
members, err := s.LANSegmentMembers(segment)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve members for segment %s - %w", segment, err)
}
var joinAddrs []string
for _, m := range members {
if ok, _ := metadata.IsConsulServer(m); ok {
serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
joinAddrs = append(joinAddrs, serfAddr.String())
}
}
return joinAddrs, nil
}
// ForwardRPC is basically just renaming forward, in a future commit I am going to actually do the rename.
func (s *Server) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
return s.forward(method, info, args, reply)
}
// GetConfig will return the servers configuration - this is needed to satisfy
// interfaces: AutoConfigDelegate
func (s *Server) GetConfig() *Config {
return s.config
}
// TLSConfigurator just returns the servers TLS Configurator - this is needed to satisfy
// interfaces: AutoConfigDelegate
func (s *Server) TLSConfigurator() *tlsutil.Configurator {
return s.tlsConfigurator
}
// peersInfoContent is used to help operators understand what happened to the
// peers.json file. This is written to a file called peers.info in the same
// location.

View File

@ -1534,3 +1534,97 @@ func TestServer_CALogging(t *testing.T) {
require.Contains(t, buf.String(), "consul CA provider configured")
}
func TestServer_DatacenterJoinAddresses(t *testing.T) {
conf := testClusterConfig{
Datacenter: "primary",
Servers: 3,
}
nodes := newTestCluster(t, &conf)
var expected []string
for _, srv := range nodes.Servers {
expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
}
actual, err := nodes.Servers[0].DatacenterJoinAddresses("")
require.NoError(t, err)
require.ElementsMatch(t, expected, actual)
}
func TestServer_CreateACLToken(t *testing.T) {
_, srv, codec := testACLServerWithConfig(t, nil, false)
waitForLeaderEstablishment(t, srv)
r1, err := upsertTestRole(codec, TestDefaultMasterToken, "dc1")
require.NoError(t, err)
t.Run("predefined-ids", func(t *testing.T) {
accessor := "554cd3ab-5d4e-4d6e-952e-4e8b6c77bfb3"
secret := "ef453f31-ad58-4ec8-8bf8-342e99763026"
in := &structs.ACLToken{
AccessorID: accessor,
SecretID: secret,
Description: "test",
Policies: []structs.ACLTokenPolicyLink{
{
ID: structs.ACLPolicyGlobalManagementID,
},
},
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "foo",
Datacenter: "bar",
},
},
ServiceIdentities: []*structs.ACLServiceIdentity{
{
ServiceName: "web",
},
},
Roles: []structs.ACLTokenRoleLink{
{
ID: r1.ID,
},
},
}
out, err := srv.CreateACLToken(in)
require.NoError(t, err)
require.Equal(t, accessor, out.AccessorID)
require.Equal(t, secret, out.SecretID)
require.Equal(t, "test", out.Description)
require.NotZero(t, out.CreateTime)
require.Len(t, out.Policies, 1)
require.Len(t, out.Roles, 1)
require.Len(t, out.NodeIdentities, 1)
require.Len(t, out.ServiceIdentities, 1)
require.Equal(t, structs.ACLPolicyGlobalManagementID, out.Policies[0].ID)
require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
require.Equal(t, "web", out.ServiceIdentities[0].ServiceName)
require.Equal(t, r1.ID, out.Roles[0].ID)
})
t.Run("autogen-ids", func(t *testing.T) {
in := &structs.ACLToken{
Description: "test",
NodeIdentities: []*structs.ACLNodeIdentity{
{
NodeName: "foo",
Datacenter: "bar",
},
},
}
out, err := srv.CreateACLToken(in)
require.NoError(t, err)
require.NotEmpty(t, out.AccessorID)
require.NotEmpty(t, out.SecretID)
require.Equal(t, "test", out.Description)
require.NotZero(t, out.CreateTime)
require.Len(t, out.NodeIdentities, 1)
require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
})
}

View File

@ -560,7 +560,7 @@ func (p *ConnPool) RPC(
// secure or insecure variant depending on whether its an ongoing
// or first time config request. For now though this is fine until
// those ongoing requests are implemented.
if method == "AutoEncrypt.Sign" || method == "Cluster.AutoConfig" {
if method == "AutoEncrypt.Sign" || method == "AutoConfig.InitialConfiguration" {
return p.rpcInsecure(dc, nodeName, addr, method, args, reply)
} else {
return p.rpc(dc, nodeName, addr, method, args, reply)