consul/command/agent/agent_test.go

1309 lines
31 KiB
Go

package agent
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sync/atomic"
"testing"
"time"
"github.com/hashicorp/consul/consul"
"github.com/hashicorp/consul/consul/structs"
"github.com/hashicorp/consul/testutil"
)
var offset uint64
func nextConfig() *Config {
idx := int(atomic.AddUint64(&offset, 1))
conf := DefaultConfig()
conf.AdvertiseAddr = "127.0.0.1"
conf.Bootstrap = true
conf.Datacenter = "dc1"
conf.NodeName = fmt.Sprintf("Node %d", idx)
conf.BindAddr = "127.0.0.1"
conf.Ports.DNS = 19000 + idx
conf.Ports.HTTP = 18800 + idx
conf.Ports.RPC = 18600 + idx
conf.Ports.SerfLan = 18200 + idx
conf.Ports.SerfWan = 18400 + idx
conf.Ports.Server = 18000 + idx
conf.Server = true
conf.ACLDatacenter = "dc1"
conf.ACLMasterToken = "root"
cons := consul.DefaultConfig()
conf.ConsulConfig = cons
cons.SerfLANConfig.MemberlistConfig.SuspicionMult = 3
cons.SerfLANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
cons.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
cons.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
cons.SerfWANConfig.MemberlistConfig.SuspicionMult = 3
cons.SerfWANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
cons.SerfWANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
cons.SerfWANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
cons.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
cons.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
cons.RaftConfig.ElectionTimeout = 40 * time.Millisecond
return conf
}
func makeAgentLog(t *testing.T, conf *Config, l io.Writer) (string, *Agent) {
dir, err := ioutil.TempDir("", "agent")
if err != nil {
t.Fatalf(fmt.Sprintf("err: %v", err))
}
conf.DataDir = dir
agent, err := Create(conf, l)
if err != nil {
os.RemoveAll(dir)
t.Fatalf(fmt.Sprintf("err: %v", err))
}
return dir, agent
}
func makeAgentKeyring(t *testing.T, conf *Config, key string) (string, *Agent) {
dir, err := ioutil.TempDir("", "agent")
if err != nil {
t.Fatalf("err: %v", err)
}
conf.DataDir = dir
fileLAN := filepath.Join(dir, serfLANKeyring)
if err := initKeyring(fileLAN, key); err != nil {
t.Fatalf("err: %s", err)
}
fileWAN := filepath.Join(dir, serfWANKeyring)
if err := initKeyring(fileWAN, key); err != nil {
t.Fatalf("err: %s", err)
}
agent, err := Create(conf, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
return dir, agent
}
func makeAgent(t *testing.T, conf *Config) (string, *Agent) {
return makeAgentLog(t, conf, nil)
}
func TestAgentStartStop(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
if err := agent.Leave(); err != nil {
t.Fatalf("err: %v", err)
}
if err := agent.Shutdown(); err != nil {
t.Fatalf("err: %v", err)
}
select {
case <-agent.ShutdownCh():
default:
t.Fatalf("should be closed")
}
}
func TestAgent_RPCPing(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
var out struct{}
if err := agent.RPC("Status.Ping", struct{}{}, &out); err != nil {
t.Fatalf("err: %v", err)
}
}
func TestAgent_AddService(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
// Service registration with a single check
{
srv := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
chkTypes := CheckTypes{
&CheckType{
TTL: time.Minute,
Notes: "redis heath check 2",
},
}
err := agent.AddService(srv, chkTypes, false, "")
if err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a state mapping
if _, ok := agent.state.Services()["redis"]; !ok {
t.Fatalf("missing redis service")
}
// Ensure the check is registered
if _, ok := agent.state.Checks()["service:redis"]; !ok {
t.Fatalf("missing redis check")
}
// Ensure a TTL is setup
if _, ok := agent.checkTTLs["service:redis"]; !ok {
t.Fatalf("missing redis check ttl")
}
// Ensure the notes are passed through
if agent.state.Checks()["service:redis"].Notes == "" {
t.Fatalf("missing redis check notes")
}
}
// Service registration with multiple checks
{
srv := &structs.NodeService{
ID: "memcache",
Service: "memcache",
Tags: []string{"bar"},
Port: 8000,
}
chkTypes := CheckTypes{
&CheckType{
TTL: time.Minute,
Notes: "memcache health check 1",
},
&CheckType{
TTL: time.Second,
Notes: "memcache heath check 2",
},
}
if err := agent.AddService(srv, chkTypes, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a state mapping
if _, ok := agent.state.Services()["memcache"]; !ok {
t.Fatalf("missing memcache service")
}
// Ensure both checks were added
if _, ok := agent.state.Checks()["service:memcache:1"]; !ok {
t.Fatalf("missing memcache:1 check")
}
if _, ok := agent.state.Checks()["service:memcache:2"]; !ok {
t.Fatalf("missing memcache:2 check")
}
// Ensure a TTL is setup
if _, ok := agent.checkTTLs["service:memcache:1"]; !ok {
t.Fatalf("missing memcache:1 check ttl")
}
if _, ok := agent.checkTTLs["service:memcache:2"]; !ok {
t.Fatalf("missing memcache:2 check ttl")
}
// Ensure the notes are passed through
if agent.state.Checks()["service:memcache:1"].Notes == "" {
t.Fatalf("missing redis check notes")
}
if agent.state.Checks()["service:memcache:2"].Notes == "" {
t.Fatalf("missing redis check notes")
}
}
}
func TestAgent_RemoveService(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
// Remove a service that doesn't exist
if err := agent.RemoveService("redis", false); err != nil {
t.Fatalf("err: %v", err)
}
// Remove the consul service
if err := agent.RemoveService("consul", false); err == nil {
t.Fatalf("should have errored")
}
// Remove without an ID
if err := agent.RemoveService("", false); err == nil {
t.Fatalf("should have errored")
}
// Removing a service with a single check works
{
srv := &structs.NodeService{
ID: "memcache",
Service: "memcache",
Port: 8000,
}
chkTypes := CheckTypes{&CheckType{TTL: time.Minute}}
if err := agent.AddService(srv, chkTypes, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Add a check after the fact with a specific check ID
check := &CheckDefinition{
ID: "check2",
Name: "check2",
ServiceID: "memcache",
CheckType: CheckType{TTL: time.Minute},
}
hc := check.HealthCheck("node1")
if err := agent.AddCheck(hc, &check.CheckType, false, ""); err != nil {
t.Fatalf("err: %s", err)
}
if err := agent.RemoveService("memcache", false); err != nil {
t.Fatalf("err: %s", err)
}
if _, ok := agent.state.Checks()["service:memcache"]; ok {
t.Fatalf("have memcache check")
}
if _, ok := agent.state.Checks()["check2"]; ok {
t.Fatalf("have check2 check")
}
}
// Removing a service with multiple checks works
{
srv := &structs.NodeService{
ID: "redis",
Service: "redis",
Port: 8000,
}
chkTypes := CheckTypes{
&CheckType{TTL: time.Minute},
&CheckType{TTL: 30 * time.Second},
}
if err := agent.AddService(srv, chkTypes, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Remove the service
if err := agent.RemoveService("redis", false); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a state mapping
if _, ok := agent.state.Services()["redis"]; ok {
t.Fatalf("have redis service")
}
// Ensure checks were removed
if _, ok := agent.state.Checks()["service:redis:1"]; ok {
t.Fatalf("check redis:1 should be removed")
}
if _, ok := agent.state.Checks()["service:redis:2"]; ok {
t.Fatalf("check redis:2 should be removed")
}
// Ensure a TTL is setup
if _, ok := agent.checkTTLs["service:redis:1"]; ok {
t.Fatalf("check ttl for redis:1 should be removed")
}
if _, ok := agent.checkTTLs["service:redis:2"]; ok {
t.Fatalf("check ttl for redis:2 should be removed")
}
}
}
func TestAgent_AddCheck(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
health := &structs.HealthCheck{
Node: "foo",
CheckID: "mem",
Name: "memory util",
Status: structs.HealthCritical,
}
chk := &CheckType{
Script: "exit 0",
Interval: 15 * time.Second,
}
err := agent.AddCheck(health, chk, false, "")
if err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a check mapping
if _, ok := agent.state.Checks()["mem"]; !ok {
t.Fatalf("missing mem check")
}
// Ensure a TTL is setup
if _, ok := agent.checkMonitors["mem"]; !ok {
t.Fatalf("missing mem monitor")
}
}
func TestAgent_AddCheck_MinInterval(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
health := &structs.HealthCheck{
Node: "foo",
CheckID: "mem",
Name: "memory util",
Status: structs.HealthCritical,
}
chk := &CheckType{
Script: "exit 0",
Interval: time.Microsecond,
}
err := agent.AddCheck(health, chk, false, "")
if err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a check mapping
if _, ok := agent.state.Checks()["mem"]; !ok {
t.Fatalf("missing mem check")
}
// Ensure a TTL is setup
if mon, ok := agent.checkMonitors["mem"]; !ok {
t.Fatalf("missing mem monitor")
} else if mon.Interval != MinInterval {
t.Fatalf("bad mem monitor interval")
}
}
func TestAgent_AddCheck_MissingService(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
health := &structs.HealthCheck{
Node: "foo",
CheckID: "baz",
Name: "baz check 1",
ServiceID: "baz",
}
chk := &CheckType{
Script: "exit 0",
Interval: time.Microsecond,
}
err := agent.AddCheck(health, chk, false, "")
if err == nil || err.Error() != `ServiceID "baz" does not exist` {
t.Fatalf("expected service id error, got: %v", err)
}
}
func TestAgent_RemoveCheck(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
// Remove check that doesn't exist
if err := agent.RemoveCheck("mem", false); err != nil {
t.Fatalf("err: %v", err)
}
// Remove without an ID
if err := agent.RemoveCheck("", false); err == nil {
t.Fatalf("should have errored")
}
health := &structs.HealthCheck{
Node: "foo",
CheckID: "mem",
Name: "memory util",
Status: structs.HealthCritical,
}
chk := &CheckType{
Script: "exit 0",
Interval: 15 * time.Second,
}
err := agent.AddCheck(health, chk, false, "")
if err != nil {
t.Fatalf("err: %v", err)
}
// Remove check
if err := agent.RemoveCheck("mem", false); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a check mapping
if _, ok := agent.state.Checks()["mem"]; ok {
t.Fatalf("have mem check")
}
// Ensure a TTL is setup
if _, ok := agent.checkMonitors["mem"]; ok {
t.Fatalf("have mem monitor")
}
}
func TestAgent_UpdateCheck(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
health := &structs.HealthCheck{
Node: "foo",
CheckID: "mem",
Name: "memory util",
Status: structs.HealthCritical,
}
chk := &CheckType{
TTL: 15 * time.Second,
}
err := agent.AddCheck(health, chk, false, "")
if err != nil {
t.Fatalf("err: %v", err)
}
// Remove check
if err := agent.UpdateCheck("mem", structs.HealthPassing, "foo"); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a check mapping
status := agent.state.Checks()["mem"]
if status.Status != structs.HealthPassing {
t.Fatalf("bad: %v", status)
}
if status.Output != "foo" {
t.Fatalf("bad: %v", status)
}
}
func TestAgent_ConsulService(t *testing.T) {
dir, agent := makeAgent(t, nextConfig())
defer os.RemoveAll(dir)
defer agent.Shutdown()
testutil.WaitForLeader(t, agent.RPC, "dc1")
// Consul service is registered
services := agent.state.Services()
if _, ok := services[consul.ConsulServiceID]; !ok {
t.Fatalf("%s service should be registered", consul.ConsulServiceID)
}
// Perform anti-entropy on consul service
if err := agent.state.syncService(consul.ConsulServiceID); err != nil {
t.Fatalf("err: %s", err)
}
// Consul service should be in sync
if !agent.state.serviceStatus[consul.ConsulServiceID].inSync {
t.Fatalf("%s service should be in sync", consul.ConsulServiceID)
}
}
func TestAgent_PersistService(t *testing.T) {
config := nextConfig()
config.Server = false
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
file := filepath.Join(agent.config.DataDir, servicesDir, stringHash(svc.ID))
// Check is not persisted unless requested
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
if _, err := os.Stat(file); err == nil {
t.Fatalf("should not persist")
}
// Persists to file if requested
if err := agent.AddService(svc, nil, true, "mytoken"); err != nil {
t.Fatalf("err: %v", err)
}
if _, err := os.Stat(file); err != nil {
t.Fatalf("err: %s", err)
}
expected, err := json.Marshal(persistedService{
Token: "mytoken",
Service: svc,
})
if err != nil {
t.Fatalf("err: %s", err)
}
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("err: %s", err)
}
if !bytes.Equal(expected, content) {
t.Fatalf("bad: %s", string(content))
}
// Updates service definition on disk
svc.Port = 8001
if err := agent.AddService(svc, nil, true, "mytoken"); err != nil {
t.Fatalf("err: %v", err)
}
expected, err = json.Marshal(persistedService{
Token: "mytoken",
Service: svc,
})
if err != nil {
t.Fatalf("err: %s", err)
}
content, err = ioutil.ReadFile(file)
if err != nil {
t.Fatalf("err: %s", err)
}
if !bytes.Equal(expected, content) {
t.Fatalf("bad: %s", string(content))
}
agent.Shutdown()
// Should load it back during later start
agent2, err := Create(config, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
defer agent2.Shutdown()
restored, ok := agent2.state.services[svc.ID]
if !ok {
t.Fatalf("bad: %#v", agent2.state.services)
}
if agent2.state.serviceTokens[svc.ID] != "mytoken" {
t.Fatalf("bad: %#v", agent2.state.services[svc.ID])
}
if restored.Port != 8001 {
t.Fatalf("bad: %#v", restored)
}
}
func TestAgent_persistedService_compat(t *testing.T) {
// Tests backwards compatibility of persisted services from pre-0.5.1
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
// Encode the NodeService directly. This is what previous versions
// would serialize to the file (without the wrapper)
encoded, err := json.Marshal(svc)
if err != nil {
t.Fatalf("err: %s", err)
}
// Write the content to the file
file := filepath.Join(agent.config.DataDir, servicesDir, stringHash(svc.ID))
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatalf("err: %s", err)
}
if err := ioutil.WriteFile(file, encoded, 0600); err != nil {
t.Fatalf("err: %s", err)
}
// Load the services
if err := agent.loadServices(config); err != nil {
t.Fatalf("err: %s", err)
}
// Ensure the service was restored
services := agent.state.Services()
result, ok := services["redis"]
if !ok {
t.Fatalf("missing service")
}
if !reflect.DeepEqual(result, svc) {
t.Fatalf("bad: %#v", result)
}
}
func TestAgent_PurgeService(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
file := filepath.Join(agent.config.DataDir, servicesDir, stringHash(svc.ID))
if err := agent.AddService(svc, nil, true, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Not removed
if err := agent.RemoveService(svc.ID, false); err != nil {
t.Fatalf("err: %s", err)
}
if _, err := os.Stat(file); err != nil {
t.Fatalf("err: %s", err)
}
// Removed
if err := agent.RemoveService(svc.ID, true); err != nil {
t.Fatalf("err: %s", err)
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("bad: %#v", err)
}
}
func TestAgent_PurgeServiceOnDuplicate(t *testing.T) {
config := nextConfig()
config.Server = false
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc1 := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
// First persist the service
if err := agent.AddService(svc1, nil, true, ""); err != nil {
t.Fatalf("err: %v", err)
}
agent.Shutdown()
// Try bringing the agent back up with the service already
// existing in the config
svc2 := &ServiceDefinition{
ID: "redis",
Name: "redis",
Tags: []string{"bar"},
Port: 9000,
}
config.Services = []*ServiceDefinition{svc2}
agent2, err := Create(config, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
defer agent2.Shutdown()
file := filepath.Join(agent.config.DataDir, servicesDir, stringHash(svc1.ID))
if _, err := os.Stat(file); err == nil {
t.Fatalf("should have removed persisted service")
}
result, ok := agent2.state.services[svc2.ID]
if !ok {
t.Fatalf("missing service registration")
}
if !reflect.DeepEqual(result.Tags, svc2.Tags) || result.Port != svc2.Port {
t.Fatalf("bad: %#v", result)
}
}
func TestAgent_PersistCheck(t *testing.T) {
config := nextConfig()
config.Server = false
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
check := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "mem",
Name: "memory check",
Status: structs.HealthPassing,
}
chkType := &CheckType{
Script: "/bin/true",
Interval: 10 * time.Second,
}
file := filepath.Join(agent.config.DataDir, checksDir, stringHash(check.CheckID))
// Not persisted if not requested
if err := agent.AddCheck(check, chkType, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
if _, err := os.Stat(file); err == nil {
t.Fatalf("should not persist")
}
// Should persist if requested
if err := agent.AddCheck(check, chkType, true, "mytoken"); err != nil {
t.Fatalf("err: %v", err)
}
if _, err := os.Stat(file); err != nil {
t.Fatalf("err: %s", err)
}
expected, err := json.Marshal(persistedCheck{
Check: check,
ChkType: chkType,
Token: "mytoken",
})
if err != nil {
t.Fatalf("err: %s", err)
}
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("err: %s", err)
}
if !bytes.Equal(expected, content) {
t.Fatalf("bad: %s", string(content))
}
// Updates the check definition on disk
check.Name = "mem1"
if err := agent.AddCheck(check, chkType, true, "mytoken"); err != nil {
t.Fatalf("err: %v", err)
}
expected, err = json.Marshal(persistedCheck{
Check: check,
ChkType: chkType,
Token: "mytoken",
})
if err != nil {
t.Fatalf("err: %s", err)
}
content, err = ioutil.ReadFile(file)
if err != nil {
t.Fatalf("err: %s", err)
}
if !bytes.Equal(expected, content) {
t.Fatalf("bad: %s", string(content))
}
agent.Shutdown()
// Should load it back during later start
agent2, err := Create(config, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
defer agent2.Shutdown()
result, ok := agent2.state.checks[check.CheckID]
if !ok {
t.Fatalf("bad: %#v", agent2.state.checks)
}
if result.Status != structs.HealthCritical {
t.Fatalf("bad: %#v", result)
}
if result.Name != "mem1" {
t.Fatalf("bad: %#v", result)
}
// Should have restored the monitor
if _, ok := agent2.checkMonitors[check.CheckID]; !ok {
t.Fatalf("bad: %#v", agent2.checkMonitors)
}
if agent2.state.checkTokens[check.CheckID] != "mytoken" {
t.Fatalf("bad: %s", agent2.state.checkTokens[check.CheckID])
}
}
func TestAgent_PurgeCheck(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
check := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "mem",
Name: "memory check",
Status: structs.HealthPassing,
}
file := filepath.Join(agent.config.DataDir, checksDir, stringHash(check.CheckID))
if err := agent.AddCheck(check, nil, true, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Not removed
if err := agent.RemoveCheck(check.CheckID, false); err != nil {
t.Fatalf("err: %s", err)
}
if _, err := os.Stat(file); err != nil {
t.Fatalf("err: %s", err)
}
// Removed
if err := agent.RemoveCheck(check.CheckID, true); err != nil {
t.Fatalf("err: %s", err)
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("bad: %#v", err)
}
}
func TestAgent_PurgeCheckOnDuplicate(t *testing.T) {
config := nextConfig()
config.Server = false
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
check1 := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "mem",
Name: "memory check",
Status: structs.HealthPassing,
}
// First persist the check
if err := agent.AddCheck(check1, nil, true, ""); err != nil {
t.Fatalf("err: %v", err)
}
agent.Shutdown()
// Start again with the check registered in config
check2 := &CheckDefinition{
ID: "mem",
Name: "memory check",
Notes: "my cool notes",
CheckType: CheckType{
Script: "/bin/check-redis.py",
Interval: 30 * time.Second,
},
}
config.Checks = []*CheckDefinition{check2}
agent2, err := Create(config, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
defer agent2.Shutdown()
file := filepath.Join(agent.config.DataDir, checksDir, stringHash(check1.CheckID))
if _, err := os.Stat(file); err == nil {
t.Fatalf("should have removed persisted check")
}
result, ok := agent2.state.checks[check2.ID]
if !ok {
t.Fatalf("missing check registration")
}
expected := check2.HealthCheck(config.NodeName)
if !reflect.DeepEqual(expected, result) {
t.Fatalf("bad: %#v", result)
}
}
func TestAgent_loadChecks_token(t *testing.T) {
config := nextConfig()
config.Checks = append(config.Checks, &CheckDefinition{
ID: "rabbitmq",
Name: "rabbitmq",
Token: "abc123",
CheckType: CheckType{
TTL: 10 * time.Second,
},
})
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
checks := agent.state.Checks()
if _, ok := checks["rabbitmq"]; !ok {
t.Fatalf("missing check")
}
if token := agent.state.CheckToken("rabbitmq"); token != "abc123" {
t.Fatalf("bad: %s", token)
}
}
func TestAgent_unloadChecks(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
// First register a service
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Register a check
check1 := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "service:redis",
Name: "redischeck",
Status: structs.HealthPassing,
ServiceID: "redis",
ServiceName: "redis",
}
if err := agent.AddCheck(check1, nil, false, ""); err != nil {
t.Fatalf("err: %s", err)
}
found := false
for check, _ := range agent.state.Checks() {
if check == check1.CheckID {
found = true
break
}
}
if !found {
t.Fatalf("check should have been registered")
}
// Unload all of the checks
if err := agent.unloadChecks(); err != nil {
t.Fatalf("err: %s", err)
}
// Make sure it was unloaded
for check, _ := range agent.state.Checks() {
if check == check1.CheckID {
t.Fatalf("should have unloaded checks")
}
}
}
func TestAgent_loadServices_token(t *testing.T) {
config := nextConfig()
config.Services = append(config.Services, &ServiceDefinition{
ID: "rabbitmq",
Name: "rabbitmq",
Port: 5672,
Token: "abc123",
})
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
services := agent.state.Services()
if _, ok := services["rabbitmq"]; !ok {
t.Fatalf("missing service")
}
if token := agent.state.ServiceToken("rabbitmq"); token != "abc123" {
t.Fatalf("bad: %s", token)
}
}
func TestAgent_unloadServices(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
// Register the service
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
found := false
for id, _ := range agent.state.Services() {
if id == svc.ID {
found = true
break
}
}
if !found {
t.Fatalf("should have registered service")
}
// Unload all services
if err := agent.unloadServices(); err != nil {
t.Fatalf("err: %s", err)
}
// Make sure it was unloaded and the consul service remains
found = false
for id, _ := range agent.state.Services() {
if id == svc.ID {
t.Fatalf("should have unloaded services")
}
if id == consul.ConsulServiceID {
found = true
}
}
if !found {
t.Fatalf("consul service should not be removed")
}
}
func TestAgent_ServiceMaintenanceMode(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
// Register the service
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Enter maintenance mode for the service
if err := agent.EnableServiceMaintenance("redis", "broken"); err != nil {
t.Fatalf("err: %s", err)
}
// Make sure the critical health check was added
checkID := serviceMaintCheckID("redis")
check, ok := agent.state.Checks()[checkID]
if !ok {
t.Fatalf("should have registered critical maintenance check")
}
// Ensure the reason was set in notes
if check.Notes != "broken" {
t.Fatalf("bad: %#v", check)
}
// Leave maintenance mode
if err := agent.DisableServiceMaintenance("redis"); err != nil {
t.Fatalf("err: %s", err)
}
// Ensure the check was deregistered
if _, ok := agent.state.Checks()[checkID]; ok {
t.Fatalf("should have deregistered maintenance check")
}
// Enter service maintenance mode without providing a reason
if err := agent.EnableServiceMaintenance("redis", ""); err != nil {
t.Fatalf("err: %s", err)
}
// Ensure the check was registered with the default notes
check, ok = agent.state.Checks()[checkID]
if !ok {
t.Fatalf("should have registered critical check")
}
if check.Notes != defaultServiceMaintReason {
t.Fatalf("bad: %#v", check)
}
}
func TestAgent_addCheck_restoresSnapshot(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
// First register a service
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Register a check
check1 := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "service:redis",
Name: "redischeck",
Status: structs.HealthPassing,
ServiceID: "redis",
ServiceName: "redis",
}
if err := agent.AddCheck(check1, nil, false, ""); err != nil {
t.Fatalf("err: %s", err)
}
// Re-registering the service preserves the state of the check
chkTypes := CheckTypes{&CheckType{TTL: 30 * time.Second}}
if err := agent.AddService(svc, chkTypes, false, ""); err != nil {
t.Fatalf("err: %s", err)
}
check, ok := agent.state.Checks()["service:redis"]
if !ok {
t.Fatalf("missing check")
}
if check.Status != structs.HealthPassing {
t.Fatalf("bad: %s", check.Status)
}
}
func TestAgent_NodeMaintenanceMode(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
// Enter maintenance mode for the node
agent.EnableNodeMaintenance("broken")
// Make sure the critical health check was added
check, ok := agent.state.Checks()[nodeMaintCheckID]
if !ok {
t.Fatalf("should have registered critical node check")
}
// Ensure the reason was set in notes
if check.Notes != "broken" {
t.Fatalf("bad: %#v", check)
}
// Leave maintenance mode
agent.DisableNodeMaintenance()
// Ensure the check was deregistered
if _, ok := agent.state.Checks()[nodeMaintCheckID]; ok {
t.Fatalf("should have deregistered critical node check")
}
// Enter maintenance mode without passing a reason
agent.EnableNodeMaintenance("")
// Make sure the check was registered with the default note
check, ok = agent.state.Checks()[nodeMaintCheckID]
if !ok {
t.Fatalf("should have registered critical node check")
}
if check.Notes != defaultNodeMaintReason {
t.Fatalf("bad: %#v", check)
}
}
func TestAgent_checkStateSnapshot(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
// First register a service
svc := &structs.NodeService{
ID: "redis",
Service: "redis",
Tags: []string{"foo"},
Port: 8000,
}
if err := agent.AddService(svc, nil, false, ""); err != nil {
t.Fatalf("err: %v", err)
}
// Register a check
check1 := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "service:redis",
Name: "redischeck",
Status: structs.HealthPassing,
ServiceID: "redis",
ServiceName: "redis",
}
if err := agent.AddCheck(check1, nil, true, ""); err != nil {
t.Fatalf("err: %s", err)
}
// Snapshot the state
snap := agent.snapshotCheckState()
// Unload all of the checks
if err := agent.unloadChecks(); err != nil {
t.Fatalf("err: %s", err)
}
// Reload the checks
if err := agent.loadChecks(config); err != nil {
t.Fatalf("err: %s", err)
}
// Restore the state
agent.restoreCheckState(snap)
// Search for the check
out, ok := agent.state.Checks()[check1.CheckID]
if !ok {
t.Fatalf("check should have been registered")
}
// Make sure state was restored
if out.Status != structs.HealthPassing {
t.Fatalf("should have restored check state")
}
}
func TestAgent_loadChecks_checkFails(t *testing.T) {
config := nextConfig()
dir, agent := makeAgent(t, config)
defer os.RemoveAll(dir)
defer agent.Shutdown()
// Persist a health check with an invalid service ID
check := &structs.HealthCheck{
Node: config.NodeName,
CheckID: "service:redis",
Name: "redischeck",
Status: structs.HealthPassing,
ServiceID: "nope",
}
if err := agent.persistCheck(check, nil); err != nil {
t.Fatalf("err: %s", err)
}
// Check to make sure the check was persisted
checkHash := stringHash(check.CheckID)
checkPath := filepath.Join(config.DataDir, checksDir, checkHash)
if _, err := os.Stat(checkPath); err != nil {
t.Fatalf("err: %s", err)
}
// Try loading the checks from the persisted files
if err := agent.loadChecks(config); err != nil {
t.Fatalf("err: %s", err)
}
// Ensure the erroneous check was purged
if _, err := os.Stat(checkPath); err == nil {
t.Fatalf("should have purged check")
}
}