Merge remote-tracking branch 'upstream/master'

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.4.1 (Unreleased)
+## 0.4.1 (October 20, 2014)
 
 FEATURES:
 
@@ -23,6 +23,7 @@ BUG FIXES:
  * Serf snapshot compaction works on Windows [GH-332]
  * Raft snapshots work on Windows [GH-265]
  * Consul service entry clean by clients now possible
+ * Fixing improper deserialization
 
 IMPROVEMENTS:
 
@@ -203,11 +204,11 @@ FEATURES:
   * /v1/health/service/ endpoint can take an optional `?passing` flag
       to filter to only nodes with passing results. [GH-57]
   * The KV endpoint suports listing keys with the `?keys` query parameter,
-      and limited up to a seperator using `?seperator=`.
+      and limited up to a separator using `?separator=`.
 
 IMPROVEMENTS:
 
-  * Health check output goes into seperate `Output` field instead
+  * Health check output goes into separate `Output` field instead
       of overriding `Notes`. [GH-59]
   * Adding a minimum check interval to prevent checks with extremely
       low intervals fork bombing. [GH-64]
@@ -226,7 +227,7 @@ BUG FIXES:
   * DNS parser can handler period in a tag name. [GH-39]
   * "application/json" content-type is sent on HTTP requests. [GH-45]
   * Work around for LMDB delete issue. [GH-85]
-  * Fixed tag gossip propogation for rapid restart. [GH-86]
+  * Fixed tag gossip propagation for rapid restart. [GH-86]
 
 MISC:
 

Makefile

@@ -21,6 +21,10 @@ test: deps
 integ:
 	go list ./... | INTEG_TESTS=yes xargs -n1 go test
 
+cover: deps
+	./scripts/verify_no_uuid.sh
+	go list ./... | xargs -n1 go test --cover
+
 format: deps
 	@echo "--> Running go fmt"
 	@go fmt $(PACKAGES)

Vagrantfile

@@ -2,9 +2,9 @@
 # vi: set ft=ruby :
 
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = "2"
+VAGRANTFILE_API_VERSION = '2'
 
-$script = <<SCRIPT
+@script = <<SCRIPT
 SRCROOT="/opt/go"
 
 # Install Go
@@ -29,23 +29,27 @@ sudo chown -R vagrant:vagrant /opt/gopath
 
 # Install git
 sudo apt-get install -y git-core
+
+# Install go tools
+go get code.google.com/p/go.tools/cmd/cover
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.provision "shell", inline: $script
+  config.vm.provision 'shell', inline: @script
+  config.vm.synced_folder '.', '/opt/gopath/src/github.com/hashicorp/consul'
 
-  ["vmware_fusion", "vmware_workstation"].each do |p|
-    config.vm.provider "p" do |v|
-      v.vmx["memsize"] = "2048"
-      v.vmx["numvcpus"] = "2"
-      v.vmx["cpuid.coresPerSocket"] = "1"
+  %w[vmware_fusion vmware_workstation].each do |_|
+    config.vm.provider 'p' do |v|
+      v.vmx['memsize'] = '2048'
+      v.vmx['numvcpus'] = '2'
+      v.vmx['cpuid.coresPerSocket'] = '1'
     end
   end
 
-  config.vm.define "64bit" do |n1|
-      n1.vm.box = "chef/ubuntu-10.04"
+  config.vm.define '64bit' do |n1|
+    n1.vm.box = 'chef/ubuntu-10.04'
   end
-  config.vm.define "32bit" do |n2|
-      n2.vm.box = "chef/ubuntu-10.04-i386"
+  config.vm.define '32bit' do |n2|
+    n2.vm.box = 'chef/ubuntu-10.04-i386'
   end
 end

acl/acl.go

@@ -46,6 +46,12 @@ type ACL interface {
 	// that deny a write.
 	KeyWritePrefix(string) bool
 
+	// ServiceWrite checks for permission to read a given service
+	ServiceWrite(string) bool
+
+	// ServiceRead checks for permission to read a given service
+	ServiceRead(string) bool
+
 	// ACLList checks for permission to list all the ACLs
 	ACLList() bool
 
@@ -73,6 +79,14 @@ func (s *StaticACL) KeyWritePrefix(string) bool {
 	return s.defaultAllow
 }
 
+func (s *StaticACL) ServiceRead(string) bool {
+	return s.defaultAllow
+}
+
+func (s *StaticACL) ServiceWrite(string) bool {
+	return s.defaultAllow
+}
+
 func (s *StaticACL) ACLList() bool {
 	return s.allowManage
 }
@@ -119,20 +133,29 @@ type PolicyACL struct {
 
 	// keyRules contains the key policies
 	keyRules *radix.Tree
+
+	// serviceRules contains the service policies
+	serviceRules map[string]string
 }
 
 // New is used to construct a policy based ACL from a set of policies
 // and a parent policy to resolve missing cases.
 func New(parent ACL, policy *Policy) (*PolicyACL, error) {
 	p := &PolicyACL{
-		parent:   parent,
-		keyRules: radix.New(),
+		parent:       parent,
+		keyRules:     radix.New(),
+		serviceRules: make(map[string]string, len(policy.Services)),
 	}
 
 	// Load the key policy
 	for _, kp := range policy.Keys {
 		p.keyRules.Insert(kp.Prefix, kp.Policy)
 	}
+
+	// Load the service policy
+	for _, sp := range policy.Services {
+		p.serviceRules[sp.Name] = sp.Policy
+	}
 	return p, nil
 }
 
@@ -205,6 +228,48 @@ func (p *PolicyACL) KeyWritePrefix(prefix string) bool {
 	return p.parent.KeyWritePrefix(prefix)
 }
 
+// ServiceRead checks if reading (discovery) of a service is allowed
+func (p *PolicyACL) ServiceRead(name string) bool {
+	// Check for an exact rule or catch-all
+	rule, ok := p.serviceRules[name]
+	if !ok {
+		rule, ok = p.serviceRules[""]
+	}
+	if ok {
+		switch rule {
+		case ServicePolicyWrite:
+			return true
+		case ServicePolicyRead:
+			return true
+		default:
+			return false
+		}
+	}
+
+	// No matching rule, use the parent.
+	return p.parent.ServiceRead(name)
+}
+
+// ServiceWrite checks if writing (registering) a service is allowed
+func (p *PolicyACL) ServiceWrite(name string) bool {
+	// Check for an exact rule or catch-all
+	rule, ok := p.serviceRules[name]
+	if !ok {
+		rule, ok = p.serviceRules[""]
+	}
+	if ok {
+		switch rule {
+		case ServicePolicyWrite:
+			return true
+		default:
+			return false
+		}
+	}
+
+	// No matching rule, use the parent.
+	return p.parent.ServiceWrite(name)
+}
+
 // ACLList checks if listing of ACLs is allowed
 func (p *PolicyACL) ACLList() bool {
 	return p.parent.ACLList()

acl/acl_test.go

@@ -41,6 +41,12 @@ func TestStaticACL(t *testing.T) {
 	if !all.KeyWrite("foobar") {
 		t.Fatalf("should allow")
 	}
+	if !all.ServiceRead("foobar") {
+		t.Fatalf("should allow")
+	}
+	if !all.ServiceWrite("foobar") {
+		t.Fatalf("should allow")
+	}
 	if all.ACLList() {
 		t.Fatalf("should not allow")
 	}
@@ -54,6 +60,12 @@ func TestStaticACL(t *testing.T) {
 	if none.KeyWrite("foobar") {
 		t.Fatalf("should not allow")
 	}
+	if none.ServiceRead("foobar") {
+		t.Fatalf("should not allow")
+	}
+	if none.ServiceWrite("foobar") {
+		t.Fatalf("should not allow")
+	}
 	if none.ACLList() {
 		t.Fatalf("should not noneow")
 	}
@@ -67,6 +79,12 @@ func TestStaticACL(t *testing.T) {
 	if !manage.KeyWrite("foobar") {
 		t.Fatalf("should allow")
 	}
+	if !manage.ServiceRead("foobar") {
+		t.Fatalf("should allow")
+	}
+	if !manage.ServiceWrite("foobar") {
+		t.Fatalf("should allow")
+	}
 	if !manage.ACLList() {
 		t.Fatalf("should allow")
 	}
@@ -96,19 +114,33 @@ func TestPolicyACL(t *testing.T) {
 				Policy: KeyPolicyRead,
 			},
 		},
+		Services: []*ServicePolicy{
+			&ServicePolicy{
+				Name:   "",
+				Policy: ServicePolicyWrite,
+			},
+			&ServicePolicy{
+				Name:   "foo",
+				Policy: ServicePolicyRead,
+			},
+			&ServicePolicy{
+				Name:   "bar",
+				Policy: ServicePolicyDeny,
+			},
+		},
 	}
 	acl, err := New(all, policy)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
-	type tcase struct {
+	type keycase struct {
 		inp         string
 		read        bool
 		write       bool
 		writePrefix bool
 	}
-	cases := []tcase{
+	cases := []keycase{
 		{"other", true, true, true},
 		{"foo/test", true, true, true},
 		{"foo/priv/test", false, false, false},
@@ -128,6 +160,26 @@ func TestPolicyACL(t *testing.T) {
 			t.Fatalf("Write prefix fail: %#v", c)
 		}
 	}
+
+	// Test the services
+	type servicecase struct {
+		inp   string
+		read  bool
+		write bool
+	}
+	scases := []servicecase{
+		{"other", true, true},
+		{"foo", true, false},
+		{"bar", false, false},
+	}
+	for _, c := range scases {
+		if c.read != acl.ServiceRead(c.inp) {
+			t.Fatalf("Read fail: %#v", c)
+		}
+		if c.write != acl.ServiceWrite(c.inp) {
+			t.Fatalf("Write fail: %#v", c)
+		}
+	}
 }
 
 func TestPolicyACL_Parent(t *testing.T) {
@@ -143,6 +195,16 @@ func TestPolicyACL_Parent(t *testing.T) {
 				Policy: KeyPolicyRead,
 			},
 		},
+		Services: []*ServicePolicy{
+			&ServicePolicy{
+				Name:   "other",
+				Policy: ServicePolicyWrite,
+			},
+			&ServicePolicy{
+				Name:   "foo",
+				Policy: ServicePolicyRead,
+			},
+		},
 	}
 	root, err := New(deny, policyRoot)
 	if err != nil {
@@ -164,19 +226,25 @@ func TestPolicyACL_Parent(t *testing.T) {
 				Policy: KeyPolicyRead,
 			},
 		},
+		Services: []*ServicePolicy{
+			&ServicePolicy{
+				Name:   "bar",
+				Policy: ServicePolicyDeny,
+			},
+		},
 	}
 	acl, err := New(root, policy)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
-	type tcase struct {
+	type keycase struct {
 		inp         string
 		read        bool
 		write       bool
 		writePrefix bool
 	}
-	cases := []tcase{
+	cases := []keycase{
 		{"other", false, false, false},
 		{"foo/test", true, true, true},
 		{"foo/priv/test", true, false, false},
@@ -194,4 +262,25 @@ func TestPolicyACL_Parent(t *testing.T) {
 			t.Fatalf("Write prefix fail: %#v", c)
 		}
 	}
+
+	// Test the services
+	type servicecase struct {
+		inp   string
+		read  bool
+		write bool
+	}
+	scases := []servicecase{
+		{"fail", false, false},
+		{"other", true, true},
+		{"foo", true, false},
+		{"bar", false, false},
+	}
+	for _, c := range scases {
+		if c.read != acl.ServiceRead(c.inp) {
+			t.Fatalf("Read fail: %#v", c)
+		}
+		if c.write != acl.ServiceWrite(c.inp) {
+			t.Fatalf("Write fail: %#v", c)
+		}
+	}
 }

acl/cache.go

@@ -26,7 +26,7 @@ type Cache struct {
 	ruleCache   *lru.Cache // Cache rules -> policy
 }
 
-// NewCache contructs a new policy and ACL cache of a given size
+// NewCache constructs a new policy and ACL cache of a given size
 func NewCache(size int, faultfn FaultFunc) (*Cache, error) {
 	if size <= 0 {
 		return nil, fmt.Errorf("Must provide positive cache size")

acl/policy.go

@@ -2,20 +2,25 @@ package acl
 
 import (
 	"fmt"
+
 	"github.com/hashicorp/hcl"
 )
 
 const (
-	KeyPolicyDeny  = "deny"
-	KeyPolicyRead  = "read"
-	KeyPolicyWrite = "write"
+	KeyPolicyDeny      = "deny"
+	KeyPolicyRead      = "read"
+	KeyPolicyWrite     = "write"
+	ServicePolicyDeny  = "deny"
+	ServicePolicyRead  = "read"
+	ServicePolicyWrite = "write"
 )
 
 // Policy is used to represent the policy specified by
 // an ACL configuration.
 type Policy struct {
-	ID   string       `hcl:"-"`
-	Keys []*KeyPolicy `hcl:"key,expand"`
+	ID       string           `hcl:"-"`
+	Keys     []*KeyPolicy     `hcl:"key,expand"`
+	Services []*ServicePolicy `hcl:"service,expand"`
 }
 
 // KeyPolicy represents a policy for a key
@@ -28,6 +33,16 @@ func (k *KeyPolicy) GoString() string {
 	return fmt.Sprintf("%#v", *k)
 }
 
+// ServicePolicy represents a policy for a service
+type ServicePolicy struct {
+	Name   string `hcl:",key"`
+	Policy string
+}
+
+func (k *ServicePolicy) GoString() string {
+	return fmt.Sprintf("%#v", *k)
+}
+
 // Parse is used to parse the specified ACL rules into an
 // intermediary set of policies, before being compiled into
 // the ACL
@@ -53,5 +68,17 @@ func Parse(rules string) (*Policy, error) {
 			return nil, fmt.Errorf("Invalid key policy: %#v", kp)
 		}
 	}
+
+	// Validate the service policy
+	for _, sp := range p.Services {
+		switch sp.Policy {
+		case ServicePolicyDeny:
+		case ServicePolicyRead:
+		case ServicePolicyWrite:
+		default:
+			return nil, fmt.Errorf("Invalid service policy: %#v", sp)
+		}
+	}
+
 	return p, nil
 }

acl/policy_test.go

@@ -18,6 +18,12 @@ key "foo/bar/" {
 }
 key "foo/bar/baz" {
 	policy = "deny"
+}
+service "" {
+	policy = "write"
+}
+service "foo" {
+	policy = "read"
 }
 	`
 	exp := &Policy{
@@ -39,6 +45,16 @@ key "foo/bar/baz" {
 				Policy: KeyPolicyDeny,
 			},
 		},
+		Services: []*ServicePolicy{
+			&ServicePolicy{
+				Name:   "",
+				Policy: ServicePolicyWrite,
+			},
+			&ServicePolicy{
+				Name:   "foo",
+				Policy: ServicePolicyRead,
+			},
+		},
 	}
 
 	out, err := Parse(inp)
@@ -66,6 +82,14 @@ func TestParse_JSON(t *testing.T) {
 		"foo/bar/baz": {
 			"policy": "deny"
 		}
+	},
+	"service": {
+		"": {
+			"policy": "write"
+		},
+		"foo": {
+			"policy": "read"
+		}
 	}
 }`
 	exp := &Policy{
@@ -87,6 +111,16 @@ func TestParse_JSON(t *testing.T) {
 				Policy: KeyPolicyDeny,
 			},
 		},
+		Services: []*ServicePolicy{
+			&ServicePolicy{
+				Name:   "",
+				Policy: ServicePolicyWrite,
+			},
+			&ServicePolicy{
+				Name:   "foo",
+				Policy: ServicePolicyRead,
+			},
+		},
 	}
 
 	out, err := Parse(inp)

bench/bench-aws.json

@@ -53,8 +53,8 @@
             "sudo mkdir /etc/consul.d",
             "sudo apt-get update",
             "sudo apt-get install unzip make",
-            "wget https://dl.bintray.com/mitchellh/consul/0.4.0_linux_amd64.zip",
-            "unzip 0.4.0_linux_amd64.zip",
+            "wget https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip",
+            "unzip 0.4.1_linux_amd64.zip",
             "sudo mv consul /usr/local/bin/consul",
             "chmod +x /usr/local/bin/consul"
         ]

bench/bench.json

@@ -47,8 +47,8 @@
             "mkdir /etc/consul.d",
             "apt-get update",
             "apt-get install unzip make",
-            "wget https://dl.bintray.com/mitchellh/consul/0.4.0_linux_amd64.zip",
-            "unzip 0.4.0_linux_amd64.zip",
+            "wget https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip",
+            "unzip 0.4.1_linux_amd64.zip",
             "mv consul /usr/local/bin/consul",
             "chmod +x /usr/local/bin/consul"
         ]

command/agent/agent.go

@@ -1,11 +1,13 @@
 package agent
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"net"
 	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 
@@ -14,6 +16,14 @@ import (
 	"github.com/hashicorp/serf/serf"
 )
 
+const (
+	// Path to save agent service definitions
+	servicesDir = "services"
+
+	// Path to save local agent checks
+	checksDir = "checks"
+)
+
 /*
  The agent is the long running process that is run on every machine.
  It exposes an RPC interface that is used by the CLI to control the
@@ -120,6 +130,7 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) {
 			Service: consul.ConsulServiceName,
 			ID:      consul.ConsulServiceID,
 			Port:    agent.config.Ports.Server,
+			Tags:    []string{},
 		}
 		agent.state.AddService(&consulService)
 	} else {
@@ -130,6 +141,14 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) {
 		return nil, err
 	}
 
+	// Load checks/services
+	if err := agent.reloadServices(config); err != nil {
+		return nil, err
+	}
+	if err := agent.reloadChecks(config); err != nil {
+		return nil, err
+	}
+
 	// Start handling events
 	go agent.handleEvents()
 
@@ -159,11 +178,6 @@ func (a *Agent) consulConfig() *consul.Config {
 	if a.config.DataDir != "" {
 		base.DataDir = a.config.DataDir
 	}
-	if a.config.EncryptKey != "" {
-		key, _ := a.config.EncryptBytes()
-		base.SerfLANConfig.MemberlistConfig.SecretKey = key
-		base.SerfWANConfig.MemberlistConfig.SecretKey = key
-	}
 	if a.config.NodeName != "" {
 		base.NodeName = a.config.NodeName
 	}
@@ -259,7 +273,13 @@ func (a *Agent) consulConfig() *consul.Config {
 
 // setupServer is used to initialize the Consul server
 func (a *Agent) setupServer() error {
-	server, err := consul.NewServer(a.consulConfig())
+	config := a.consulConfig()
+
+	if err := a.setupKeyrings(config); err != nil {
+		return fmt.Errorf("Failed to configure keyring: %v", err)
+	}
+
+	server, err := consul.NewServer(config)
 	if err != nil {
 		return fmt.Errorf("Failed to start Consul server: %v", err)
 	}
@@ -269,7 +289,13 @@ func (a *Agent) setupServer() error {
 
 // setupClient is used to initialize the Consul client
 func (a *Agent) setupClient() error {
-	client, err := consul.NewClient(a.consulConfig())
+	config := a.consulConfig()
+
+	if err := a.setupKeyrings(config); err != nil {
+		return fmt.Errorf("Failed to configure keyring: %v", err)
+	}
+
+	client, err := consul.NewClient(config)
 	if err != nil {
 		return fmt.Errorf("Failed to start Consul client: %v", err)
 	}
@@ -277,6 +303,47 @@ func (a *Agent) setupClient() error {
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup
+func (a *Agent) setupKeyrings(config *consul.Config) error {
+	fileLAN := filepath.Join(a.config.DataDir, serfLANKeyring)
+	fileWAN := filepath.Join(a.config.DataDir, serfWANKeyring)
+
+	if a.config.EncryptKey == "" {
+		goto LOAD
+	}
+	if _, err := os.Stat(fileLAN); err != nil {
+		if err := initKeyring(fileLAN, a.config.EncryptKey); err != nil {
+			return err
+		}
+	}
+	if a.config.Server {
+		if _, err := os.Stat(fileWAN); err != nil {
+			if err := initKeyring(fileWAN, a.config.EncryptKey); err != nil {
+				return err
+			}
+		}
+	}
+
+LOAD:
+	if _, err := os.Stat(fileLAN); err == nil {
+		config.SerfLANConfig.KeyringFile = fileLAN
+	}
+	if err := loadKeyringFile(config.SerfLANConfig); err != nil {
+		return err
+	}
+	if a.config.Server {
+		if _, err := os.Stat(fileWAN); err == nil {
+			config.SerfWANConfig.KeyringFile = fileWAN
+		}
+		if err := loadKeyringFile(config.SerfWANConfig); err != nil {
+			return err
+		}
+	}
+
+	// Success!
+	return nil
+}
+
 // RPC is used to make an RPC call to the Consul servers
 // This allows the agent to implement the Consul.Interface
 func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {
@@ -296,7 +363,7 @@ func (a *Agent) Leave() error {
 }
 
 // Shutdown is used to hard stop the agent. Should be
-// preceeded by a call to Leave to do it gracefully.
+// preceded by a call to Leave to do it gracefully.
 func (a *Agent) Shutdown() error {
 	a.shutdownLock.Lock()
 	defer a.shutdownLock.Unlock()
@@ -422,10 +489,167 @@ func (a *Agent) ResumeSync() {
 	a.state.Resume()
 }
 
+// persistService saves a service definition to a JSON file in the data dir
+func (a *Agent) persistService(service *structs.NodeService) error {
+	svcPath := filepath.Join(a.config.DataDir, servicesDir, service.ID)
+	if _, err := os.Stat(svcPath); os.IsNotExist(err) {
+		encoded, err := json.Marshal(service)
+		if err != nil {
+			return nil
+		}
+		if err := os.MkdirAll(filepath.Dir(svcPath), 0700); err != nil {
+			return err
+		}
+		fh, err := os.OpenFile(svcPath, os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			return err
+		}
+		defer fh.Close()
+		if _, err := fh.Write(encoded); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// purgeService removes a persisted service definition file from the data dir
+func (a *Agent) purgeService(serviceID string) error {
+	svcPath := filepath.Join(a.config.DataDir, servicesDir, serviceID)
+	if _, err := os.Stat(svcPath); err == nil {
+		return os.Remove(svcPath)
+	}
+	return nil
+}
+
+// restoreServices is used to load previously persisted service definitions
+// into the agent during startup.
+func (a *Agent) restoreServices() error {
+	svcDir := filepath.Join(a.config.DataDir, servicesDir)
+	if _, err := os.Stat(svcDir); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(svcDir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.Name() == servicesDir {
+			return nil
+		}
+		fh, err := os.Open(filepath.Join(svcDir, fi.Name()))
+		if err != nil {
+			return err
+		}
+		content := make([]byte, fi.Size())
+		if _, err := fh.Read(content); err != nil {
+			return err
+		}
+
+		var svc *structs.NodeService
+		if err := json.Unmarshal(content, &svc); err != nil {
+			return err
+		}
+
+		if _, ok := a.state.services[svc.ID]; ok {
+			// Purge previously persisted service. This allows config to be
+			// preferred over services persisted from the API.
+			a.logger.Printf("[DEBUG] Service %s exists, not restoring", svc.ID)
+			return a.purgeService(svc.ID)
+		} else {
+			a.logger.Printf("[DEBUG] Restored service definition: %s", svc.ID)
+			return a.AddService(svc, nil, false)
+		}
+	})
+	return err
+}
+
+// persistCheck saves a check definition to the local agent's state directory
+func (a *Agent) persistCheck(check *structs.HealthCheck, chkType *CheckType) error {
+	checkPath := filepath.Join(a.config.DataDir, checksDir, check.CheckID)
+	if _, err := os.Stat(checkPath); !os.IsNotExist(err) {
+		return err
+	}
+
+	// Create the persisted check
+	p := persistedCheck{check, chkType}
+
+	encoded, err := json.Marshal(p)
+	if err != nil {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(checkPath), 0700); err != nil {
+		return err
+	}
+	fh, err := os.OpenFile(checkPath, os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+	if _, err := fh.Write(encoded); err != nil {
+		return err
+	}
+	return nil
+}
+
+// purgeCheck removes a persisted check definition file from the data dir
+func (a *Agent) purgeCheck(checkID string) error {
+	checkPath := filepath.Join(a.config.DataDir, checksDir, checkID)
+	if _, err := os.Stat(checkPath); err == nil {
+		return os.Remove(checkPath)
+	}
+	return nil
+}
+
+// restoreChecks is used to load previously persisted health check definitions
+// into the agent during startup.
+func (a *Agent) restoreChecks() error {
+	checkDir := filepath.Join(a.config.DataDir, checksDir)
+	if _, err := os.Stat(checkDir); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(checkDir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.Name() == checksDir {
+			return nil
+		}
+		fh, err := os.Open(filepath.Join(checkDir, fi.Name()))
+		if err != nil {
+			return err
+		}
+		content := make([]byte, fi.Size())
+		if _, err := fh.Read(content); err != nil {
+			return err
+		}
+
+		var p persistedCheck
+		if err := json.Unmarshal(content, &p); err != nil {
+			return err
+		}
+
+		if _, ok := a.state.checks[p.Check.CheckID]; ok {
+			// Purge previously persisted check. This allows config to be
+			// preferred over persisted checks from the API.
+			a.logger.Printf("[DEBUG] Check %s exists, not restoring", p.Check.CheckID)
+			return a.purgeCheck(p.Check.CheckID)
+		} else {
+			// Default check to critical to avoid placing potentially unhealthy
+			// services into the active pool
+			p.Check.Status = structs.HealthCritical
+
+			a.logger.Printf("[DEBUG] Restored health check: %s", p.Check.CheckID)
+			return a.AddCheck(p.Check, p.ChkType, false)
+		}
+	})
+	return err
+}
+
 // AddService is used to add a service entry.
 // This entry is persistent and the agent will make a best effort to
 // ensure it is registered
-func (a *Agent) AddService(service *structs.NodeService, chkType *CheckType) error {
+func (a *Agent) AddService(service *structs.NodeService, chkType *CheckType, persist bool) error {
 	if service.Service == "" {
 		return fmt.Errorf("Service name missing")
 	}
@@ -439,6 +663,13 @@ func (a *Agent) AddService(service *structs.NodeService, chkType *CheckType) err
 	// Add the service
 	a.state.AddService(service)
 
+	// Persist the service to a file
+	if persist {
+		if err := a.persistService(service); err != nil {
+			return err
+		}
+	}
+
 	// Create an associated health check
 	if chkType != nil {
 		check := &structs.HealthCheck{
@@ -446,11 +677,11 @@ func (a *Agent) AddService(service *structs.NodeService, chkType *CheckType) err
 			CheckID:     fmt.Sprintf("service:%s", service.ID),
 			Name:        fmt.Sprintf("Service '%s' check", service.Service),
 			Status:      structs.HealthCritical,
-			Notes:       "",
+			Notes:       chkType.Notes,
 			ServiceID:   service.ID,
 			ServiceName: service.Service,
 		}
-		if err := a.AddCheck(check, chkType); err != nil {
+		if err := a.AddCheck(check, chkType, persist); err != nil {
 			return err
 		}
 	}
@@ -459,7 +690,7 @@ func (a *Agent) AddService(service *structs.NodeService, chkType *CheckType) err
 
 // RemoveService is used to remove a service entry.
 // The agent will make a best effort to ensure it is deregistered
-func (a *Agent) RemoveService(serviceID string) error {
+func (a *Agent) RemoveService(serviceID string, persist bool) error {
 	// Protect "consul" service from deletion by a user
 	if a.server != nil && serviceID == consul.ConsulServiceID {
 		return fmt.Errorf(
@@ -470,16 +701,23 @@ func (a *Agent) RemoveService(serviceID string) error {
 	// Remove service immeidately
 	a.state.RemoveService(serviceID)
 
+	// Remove the service from the data dir
+	if persist {
+		if err := a.purgeService(serviceID); err != nil {
+			return err
+		}
+	}
+
 	// Deregister any associated health checks
 	checkID := fmt.Sprintf("service:%s", serviceID)
-	return a.RemoveCheck(checkID)
+	return a.RemoveCheck(checkID, persist)
 }
 
 // AddCheck is used to add a health check to the agent.
 // This entry is persistent and the agent will make a best effort to
 // ensure it is registered. The Check may include a CheckType which
 // is used to automatically update the check status
-func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *CheckType) error {
+func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *CheckType, persist bool) error {
 	if check.CheckID == "" {
 		return fmt.Errorf("CheckID missing")
 	}
@@ -530,12 +768,18 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *CheckType) error {
 
 	// Add to the local state for anti-entropy
 	a.state.AddCheck(check)
+
+	// Persist the check
+	if persist {
+		return a.persistCheck(check, chkType)
+	}
+
 	return nil
 }
 
 // RemoveCheck is used to remove a health check.
 // The agent will make a best effort to ensure it is deregistered
-func (a *Agent) RemoveCheck(checkID string) error {
+func (a *Agent) RemoveCheck(checkID string, persist bool) error {
 	// Add to the local state for anti-entropy
 	a.state.RemoveCheck(checkID)
 
@@ -551,6 +795,9 @@ func (a *Agent) RemoveCheck(checkID string) error {
 		check.Stop()
 		delete(a.checkTTLs, checkID)
 	}
+	if persist {
+		return a.purgeCheck(checkID)
+	}
 	return nil
 }
 
@@ -647,3 +894,58 @@ func (a *Agent) deletePid() error {
 	}
 	return nil
 }
+
+// reloadServices reloads all known services from config and state. It is used
+// at initial agent startup as well as during config reloads.
+func (a *Agent) reloadServices(conf *Config) error {
+	for _, service := range a.state.Services() {
+		if service.ID == consul.ConsulServiceID {
+			continue
+		}
+		if err := a.RemoveService(service.ID, false); err != nil {
+			return fmt.Errorf("Failed deregistering service '%s': %v", service.ID, err)
+		}
+	}
+
+	// Register the services from config
+	for _, service := range conf.Services {
+		ns := service.NodeService()
+		chkType := service.CheckType()
+		if err := a.AddService(ns, chkType, false); err != nil {
+			return fmt.Errorf("Failed to register service '%s': %v", service.ID, err)
+		}
+	}
+
+	// Load any persisted services
+	if err := a.restoreServices(); err != nil {
+		return fmt.Errorf("Failed restoring services: %s", err)
+	}
+
+	return nil
+}
+
+// reloadChecks reloads all known checks from config and state. It can be used
+// during initial agent start or for config reloads.
+func (a *Agent) reloadChecks(conf *Config) error {
+	for _, check := range a.state.Checks() {
+		if err := a.RemoveCheck(check.CheckID, false); err != nil {
+			return fmt.Errorf("Failed deregistering check '%s': %s", check.CheckID, err)
+		}
+	}
+
+	// Register the checks from config
+	for _, check := range conf.Checks {
+		health := check.HealthCheck(conf.NodeName)
+		chkType := &check.CheckType
+		if err := a.AddCheck(health, chkType, false); err != nil {
+			return fmt.Errorf("Failed to register check '%s': %v %v", check.Name, err, check)
+		}
+	}
+
+	// Load any persisted checks
+	if err := a.restoreChecks(); err != nil {
+		return fmt.Errorf("Failed restoring checks: %s", err)
+	}
+
+	return nil
+}

command/agent/agent_endpoint.go

@@ -97,12 +97,12 @@ func (s *HTTPServer) AgentRegisterCheck(resp http.ResponseWriter, req *http.Requ
 	}
 
 	// Add the check
-	return nil, s.agent.AddCheck(health, chkType)
+	return nil, s.agent.AddCheck(health, chkType, true)
 }
 
 func (s *HTTPServer) AgentDeregisterCheck(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	checkID := strings.TrimPrefix(req.URL.Path, "/v1/agent/check/deregister/")
-	return nil, s.agent.RemoveCheck(checkID)
+	return nil, s.agent.RemoveCheck(checkID, true)
 }
 
 func (s *HTTPServer) AgentCheckPass(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
@@ -169,10 +169,10 @@ func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Re
 	}
 
 	// Add the check
-	return nil, s.agent.AddService(ns, chkType)
+	return nil, s.agent.AddService(ns, chkType, true)
 }
 
 func (s *HTTPServer) AgentDeregisterService(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	serviceID := strings.TrimPrefix(req.URL.Path, "/v1/agent/service/deregister/")
-	return nil, s.agent.RemoveService(serviceID)
+	return nil, s.agent.RemoveService(serviceID, true)
 }

command/agent/agent_endpoint_test.go

@@ -288,7 +288,7 @@ func TestHTTPAgentDeregisterCheck(t *testing.T) {
 	defer srv.agent.Shutdown()
 
 	chk := &structs.HealthCheck{Name: "test", CheckID: "test"}
-	if err := srv.agent.AddCheck(chk, nil); err != nil {
+	if err := srv.agent.AddCheck(chk, nil, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -320,7 +320,7 @@ func TestHTTPAgentPassCheck(t *testing.T) {
 
 	chk := &structs.HealthCheck{Name: "test", CheckID: "test"}
 	chkType := &CheckType{TTL: 15 * time.Second}
-	if err := srv.agent.AddCheck(chk, chkType); err != nil {
+	if err := srv.agent.AddCheck(chk, chkType, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -353,7 +353,7 @@ func TestHTTPAgentWarnCheck(t *testing.T) {
 
 	chk := &structs.HealthCheck{Name: "test", CheckID: "test"}
 	chkType := &CheckType{TTL: 15 * time.Second}
-	if err := srv.agent.AddCheck(chk, chkType); err != nil {
+	if err := srv.agent.AddCheck(chk, chkType, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -386,7 +386,7 @@ func TestHTTPAgentFailCheck(t *testing.T) {
 
 	chk := &structs.HealthCheck{Name: "test", CheckID: "test"}
 	chkType := &CheckType{TTL: 15 * time.Second}
-	if err := srv.agent.AddCheck(chk, chkType); err != nil {
+	if err := srv.agent.AddCheck(chk, chkType, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -465,7 +465,7 @@ func TestHTTPAgentDeregisterService(t *testing.T) {
 		ID:      "test",
 		Service: "test",
 	}
-	if err := srv.agent.AddService(service, nil); err != nil {
+	if err := srv.agent.AddService(service, nil, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 

command/agent/agent_test.go

@@ -1,15 +1,21 @@
 package agent
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
-	"github.com/hashicorp/consul/consul"
-	"github.com/hashicorp/consul/consul/structs"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"reflect"
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/hashicorp/consul/consul"
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/testutil"
 )
 
 var offset uint64
@@ -23,12 +29,12 @@ func nextConfig() *Config {
 	conf.Datacenter = "dc1"
 	conf.NodeName = fmt.Sprintf("Node %d", idx)
 	conf.BindAddr = "127.0.0.1"
-	conf.Ports.DNS = 18600 + idx
-	conf.Ports.HTTP = 18500 + idx
-	conf.Ports.RPC = 18400 + idx
+	conf.Ports.DNS = 19000 + idx
+	conf.Ports.HTTP = 18800 + idx
+	conf.Ports.RPC = 18600 + idx
 	conf.Ports.SerfLan = 18200 + idx
-	conf.Ports.SerfWan = 18300 + idx
-	conf.Ports.Server = 18100 + idx
+	conf.Ports.SerfWan = 18400 + idx
+	conf.Ports.Server = 18000 + idx
 	conf.Server = true
 	conf.ACLDatacenter = "dc1"
 	conf.ACLMasterToken = "root"
@@ -69,6 +75,31 @@ func makeAgentLog(t *testing.T, conf *Config, l io.Writer) (string, *Agent) {
 	return dir, agent
 }
 
+func makeAgentKeyring(t *testing.T, conf *Config, key string) (string, *Agent) {
+	dir, err := ioutil.TempDir("", "agent")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	conf.DataDir = dir
+
+	fileLAN := filepath.Join(dir, serfLANKeyring)
+	if err := initKeyring(fileLAN, key); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	fileWAN := filepath.Join(dir, serfWANKeyring)
+	if err := initKeyring(fileWAN, key); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	agent, err := Create(conf, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return dir, agent
+}
+
 func makeAgent(t *testing.T, conf *Config) (string, *Agent) {
 	return makeAgentLog(t, conf, nil)
 }
@@ -114,8 +145,11 @@ func TestAgent_AddService(t *testing.T) {
 		Tags:    []string{"foo"},
 		Port:    8000,
 	}
-	chk := &CheckType{TTL: time.Minute}
-	err := agent.AddService(srv, chk)
+	chk := &CheckType{
+		TTL:   time.Minute,
+		Notes: "redis health check",
+	}
+	err := agent.AddService(srv, chk, false)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -134,6 +168,11 @@ func TestAgent_AddService(t *testing.T) {
 	if _, ok := agent.checkTTLs["service:redis"]; !ok {
 		t.Fatalf("missing redis check ttl")
 	}
+
+	// Ensure the notes are passed through
+	if agent.state.Checks()["service:redis"].Notes == "" {
+		t.Fatalf("missing redis check notes")
+	}
 }
 
 func TestAgent_RemoveService(t *testing.T) {
@@ -142,12 +181,12 @@ func TestAgent_RemoveService(t *testing.T) {
 	defer agent.Shutdown()
 
 	// Remove a service that doesn't exist
-	if err := agent.RemoveService("redis"); err != nil {
+	if err := agent.RemoveService("redis", false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Remove the consul service
-	if err := agent.RemoveService("consul"); err == nil {
+	if err := agent.RemoveService("consul", false); err == nil {
 		t.Fatalf("should have errored")
 	}
 
@@ -157,12 +196,12 @@ func TestAgent_RemoveService(t *testing.T) {
 		Port:    8000,
 	}
 	chk := &CheckType{TTL: time.Minute}
-	if err := agent.AddService(srv, chk); err != nil {
+	if err := agent.AddService(srv, chk, false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Remove the service
-	if err := agent.RemoveService("redis"); err != nil {
+	if err := agent.RemoveService("redis", false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -197,7 +236,7 @@ func TestAgent_AddCheck(t *testing.T) {
 		Script:   "exit 0",
 		Interval: 15 * time.Second,
 	}
-	err := agent.AddCheck(health, chk)
+	err := agent.AddCheck(health, chk, false)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -228,7 +267,7 @@ func TestAgent_AddCheck_MinInterval(t *testing.T) {
 		Script:   "exit 0",
 		Interval: time.Microsecond,
 	}
-	err := agent.AddCheck(health, chk)
+	err := agent.AddCheck(health, chk, false)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -252,7 +291,7 @@ func TestAgent_RemoveCheck(t *testing.T) {
 	defer agent.Shutdown()
 
 	// Remove check that doesn't exist
-	if err := agent.RemoveCheck("mem"); err != nil {
+	if err := agent.RemoveCheck("mem", false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -266,13 +305,13 @@ func TestAgent_RemoveCheck(t *testing.T) {
 		Script:   "exit 0",
 		Interval: 15 * time.Second,
 	}
-	err := agent.AddCheck(health, chk)
+	err := agent.AddCheck(health, chk, false)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Remove check
-	if err := agent.RemoveCheck("mem"); err != nil {
+	if err := agent.RemoveCheck("mem", false); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -301,7 +340,7 @@ func TestAgent_UpdateCheck(t *testing.T) {
 	chk := &CheckType{
 		TTL: 15 * time.Second,
 	}
-	err := agent.AddCheck(health, chk)
+	err := agent.AddCheck(health, chk, false)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -326,8 +365,328 @@ func TestAgent_ConsulService(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
+	testutil.WaitForLeader(t, agent.RPC, "dc1")
+
+	// Consul service is registered
 	services := agent.state.Services()
 	if _, ok := services[consul.ConsulServiceID]; !ok {
 		t.Fatalf("%s service should be registered", consul.ConsulServiceID)
 	}
+
+	// Perform anti-entropy on consul service
+	if err := agent.state.syncService(consul.ConsulServiceID); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Consul service should be in sync
+	if !agent.state.serviceStatus[consul.ConsulServiceID].inSync {
+		t.Fatalf("%s service should be in sync", consul.ConsulServiceID)
+	}
+}
+
+func TestAgent_PersistService(t *testing.T) {
+	config := nextConfig()
+	config.Server = false
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	svc := &structs.NodeService{
+		ID:      "redis",
+		Service: "redis",
+		Tags:    []string{"foo"},
+		Port:    8000,
+	}
+
+	file := filepath.Join(agent.config.DataDir, servicesDir, svc.ID)
+
+	// Check is not persisted unless requested
+	if err := agent.AddService(svc, nil, false); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("should not persist")
+	}
+
+	// Persists to file if requested
+	if err := agent.AddService(svc, nil, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, err := os.Stat(file); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected, err := json.Marshal(svc)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if !bytes.Equal(expected, content) {
+		t.Fatalf("bad: %s", string(content))
+	}
+	agent.Shutdown()
+
+	// Should load it back during later start
+	agent2, err := Create(config, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer agent2.Shutdown()
+
+	if _, ok := agent2.state.services[svc.ID]; !ok {
+		t.Fatalf("bad: %#v", agent2.state.services)
+	}
+}
+
+func TestAgent_PurgeService(t *testing.T) {
+	config := nextConfig()
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	svc := &structs.NodeService{
+		ID:      "redis",
+		Service: "redis",
+		Tags:    []string{"foo"},
+		Port:    8000,
+	}
+
+	file := filepath.Join(agent.config.DataDir, servicesDir, svc.ID)
+	if err := agent.AddService(svc, nil, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Not removed
+	if err := agent.RemoveService(svc.ID, false); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if _, err := os.Stat(file); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Removed
+	if err := agent.RemoveService(svc.ID, true); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if _, err := os.Stat(file); !os.IsNotExist(err) {
+		t.Fatalf("bad: %#v", err)
+	}
+}
+
+func TestAgent_PurgeServiceOnDuplicate(t *testing.T) {
+	config := nextConfig()
+	config.Server = false
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	svc1 := &structs.NodeService{
+		ID:      "redis",
+		Service: "redis",
+		Tags:    []string{"foo"},
+		Port:    8000,
+	}
+
+	// First persist the service
+	if err := agent.AddService(svc1, nil, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	agent.Shutdown()
+
+	// Try bringing the agent back up with the service already
+	// existing in the config
+	svc2 := &ServiceDefinition{
+		ID:   "redis",
+		Name: "redis",
+		Tags: []string{"bar"},
+		Port: 9000,
+	}
+
+	config.Services = []*ServiceDefinition{svc2}
+	agent2, err := Create(config, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer agent2.Shutdown()
+
+	file := filepath.Join(agent.config.DataDir, servicesDir, svc1.ID)
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("should have removed persisted service")
+	}
+	result, ok := agent2.state.services[svc2.ID]
+	if !ok {
+		t.Fatalf("missing service registration")
+	}
+	if !reflect.DeepEqual(result.Tags, svc2.Tags) || result.Port != svc2.Port {
+		t.Fatalf("bad: %#v", result)
+	}
+}
+
+func TestAgent_PersistCheck(t *testing.T) {
+	config := nextConfig()
+	config.Server = false
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	check := &structs.HealthCheck{
+		Node:        config.NodeName,
+		CheckID:     "service:redis1",
+		Name:        "redischeck",
+		Status:      structs.HealthPassing,
+		ServiceID:   "redis",
+		ServiceName: "redis",
+	}
+	chkType := &CheckType{
+		Script:   "/bin/true",
+		Interval: 10 * time.Second,
+	}
+
+	file := filepath.Join(agent.config.DataDir, checksDir, check.CheckID)
+
+	// Not persisted if not requested
+	if err := agent.AddCheck(check, chkType, false); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("should not persist")
+	}
+
+	// Should persist if requested
+	if err := agent.AddCheck(check, chkType, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, err := os.Stat(file); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	p := persistedCheck{check, chkType}
+	expected, err := json.Marshal(p)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if !bytes.Equal(expected, content) {
+		t.Fatalf("bad: %s", string(content))
+	}
+	agent.Shutdown()
+
+	// Should load it back during later start
+	agent2, err := Create(config, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer agent2.Shutdown()
+
+	result, ok := agent2.state.checks[p.Check.CheckID]
+	if !ok {
+		t.Fatalf("bad: %#v", agent2.state.checks)
+	}
+	if result.Status != structs.HealthCritical {
+		t.Fatalf("bad: %#v", result)
+	}
+
+	// Should have restored the monitor
+	if _, ok := agent2.checkMonitors[p.Check.CheckID]; !ok {
+		t.Fatalf("bad: %#v", agent2.checkMonitors)
+	}
+}
+
+func TestAgent_PurgeCheck(t *testing.T) {
+	config := nextConfig()
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	check := &structs.HealthCheck{
+		Node:        config.NodeName,
+		CheckID:     "service:redis1",
+		Name:        "redischeck",
+		Status:      structs.HealthPassing,
+		ServiceID:   "redis",
+		ServiceName: "redis",
+	}
+
+	file := filepath.Join(agent.config.DataDir, checksDir, check.CheckID)
+	if err := agent.AddCheck(check, nil, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Not removed
+	if err := agent.RemoveCheck(check.CheckID, false); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if _, err := os.Stat(file); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Removed
+	if err := agent.RemoveCheck(check.CheckID, true); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if _, err := os.Stat(file); !os.IsNotExist(err) {
+		t.Fatalf("bad: %#v", err)
+	}
+}
+
+func TestAgent_PurgeCheckOnDuplicate(t *testing.T) {
+	config := nextConfig()
+	config.Server = false
+	dir, agent := makeAgent(t, config)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	check1 := &structs.HealthCheck{
+		Node:        config.NodeName,
+		CheckID:     "service:redis1",
+		Name:        "redischeck",
+		Status:      structs.HealthPassing,
+		ServiceID:   "redis",
+		ServiceName: "redis",
+	}
+
+	// First persist the check
+	if err := agent.AddCheck(check1, nil, true); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	agent.Shutdown()
+
+	// Start again with the check registered in config
+	check2 := &CheckDefinition{
+		ID:    "service:redis1",
+		Name:  "redischeck",
+		Notes: "my cool notes",
+		CheckType: CheckType{
+			Script:   "/bin/check-redis.py",
+			Interval: 30 * time.Second,
+		},
+	}
+
+	config.Checks = []*CheckDefinition{check2}
+	agent2, err := Create(config, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer agent2.Shutdown()
+
+	file := filepath.Join(agent.config.DataDir, checksDir, check1.CheckID)
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("should have removed persisted check")
+	}
+	result, ok := agent2.state.checks[check2.ID]
+	if !ok {
+		t.Fatalf("missing check registration")
+	}
+	expected := check2.HealthCheck(config.NodeName)
+	if !reflect.DeepEqual(expected, result) {
+		t.Fatalf("bad: %#v", result)
+	}
 }

command/agent/catalog_endpoint_test.go

@@ -39,6 +39,17 @@ func TestCatalogRegister(t *testing.T) {
 	if res != true {
 		t.Fatalf("bad: %v", res)
 	}
+
+	// Service should be in sync
+	if err := srv.agent.state.syncService("foo"); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if _, ok := srv.agent.state.serviceStatus["foo"]; !ok {
+		t.Fatalf("bad: %#v", srv.agent.state.serviceStatus)
+	}
+	if !srv.agent.state.serviceStatus["foo"].inSync {
+		t.Fatalf("should be in sync")
+	}
 }
 
 func TestCatalogDeregister(t *testing.T) {

command/agent/check.go

@@ -30,6 +30,8 @@ type CheckType struct {
 	Interval time.Duration
 
 	TTL time.Duration
+
+	Notes string
 }
 
 // Valid checks if the CheckType is valid
@@ -232,3 +234,10 @@ func (c *CheckTTL) SetStatus(status, output string) {
 	c.Notify.UpdateCheck(c.CheckID, status, output)
 	c.timer.Reset(c.TTL)
 }
+
+// persistedCheck is used to serialize a check and write it to disk
+// so that it may be restored later on.
+type persistedCheck struct {
+	Check   *structs.HealthCheck
+	ChkType *CheckType
+}

command/agent/command.go

@@ -43,7 +43,7 @@ type Command struct {
 	logOutput         io.Writer
 	agent             *Agent
 	rpcServer         *AgentRPC
-	httpServer        *HTTPServer
+	httpServers       []*HTTPServer
 	dnsServer         *DNSServer
 }
 
@@ -53,6 +53,7 @@ func (c *Command) readConfig() *Config {
 	var cmdConfig Config
 	var configFiles []string
 	var retryInterval string
+	var retryIntervalWan string
 	cmdFlags := flag.NewFlagSet("agent", flag.ContinueOnError)
 	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
 
@@ -71,7 +72,7 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.BoolVar(&cmdConfig.Bootstrap, "bootstrap", false, "enable server bootstrap mode")
 	cmdFlags.IntVar(&cmdConfig.BootstrapExpect, "bootstrap-expect", 0, "enable automatic bootstrap via expect mode")
 
-	cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, RPC)")
+	cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, HTTPS, RPC)")
 	cmdFlags.StringVar(&cmdConfig.BindAddr, "bind", "", "address to bind server listeners to")
 	cmdFlags.StringVar(&cmdConfig.AdvertiseAddr, "advertise", "", "address to advertise instead of bind addr")
 
@@ -83,12 +84,20 @@ func (c *Command) readConfig() *Config {
 		"enable re-joining after a previous leave")
 	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.StartJoin), "join",
 		"address of agent to join on startup")
+	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.StartJoinWan), "join-wan",
+		"address of agent to join -wan on startup")
 	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.RetryJoin), "retry-join",
 		"address of agent to join on startup with retry")
 	cmdFlags.IntVar(&cmdConfig.RetryMaxAttempts, "retry-max", 0,
 		"number of retries for joining")
 	cmdFlags.StringVar(&retryInterval, "retry-interval", "",
 		"interval between join attempts")
+	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.RetryJoinWan), "retry-join-wan",
+		"address of agent to join -wan on startup with retry")
+	cmdFlags.IntVar(&cmdConfig.RetryMaxAttemptsWan, "retry-max-wan", 0,
+		"number of retries for joining -wan")
+	cmdFlags.StringVar(&retryIntervalWan, "retry-interval-wan", "",
+		"interval between join -wan attempts")
 
 	if err := cmdFlags.Parse(c.args); err != nil {
 		return nil
@@ -103,6 +112,15 @@ func (c *Command) readConfig() *Config {
 		cmdConfig.RetryInterval = dur
 	}
 
+	if retryIntervalWan != "" {
+		dur, err := time.ParseDuration(retryIntervalWan)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error: %s", err))
+			return nil
+		}
+		cmdConfig.RetryIntervalWan = dur
+	}
+
 	config := DefaultConfig()
 	if len(configFiles) > 0 {
 		fileConfig, err := ReadConfigPaths(configFiles)
@@ -125,17 +143,27 @@ func (c *Command) readConfig() *Config {
 		config.NodeName = hostname
 	}
 
+	// Ensure we have a data directory
+	if config.DataDir == "" {
+		c.Ui.Error("Must specify data directory using -data-dir")
+		return nil
+	}
+
 	if config.EncryptKey != "" {
 		if _, err := config.EncryptBytes(); err != nil {
 			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
 			return nil
 		}
-	}
-
-	// Ensure we have a data directory
-	if config.DataDir == "" {
-		c.Ui.Error("Must specify data directory using -data-dir")
-		return nil
+		keyfileLAN := filepath.Join(config.DataDir, serfLANKeyring)
+		if _, err := os.Stat(keyfileLAN); err == nil {
+			c.Ui.Error("WARNING: LAN keyring exists but -encrypt given, ignoring")
+		}
+		if config.Server {
+			keyfileWAN := filepath.Join(config.DataDir, serfWANKeyring)
+			if _, err := os.Stat(keyfileWAN); err == nil {
+				c.Ui.Error("WARNING: WAN keyring exists but -encrypt given, ignoring")
+			}
+		}
 	}
 
 	// Verify data center is valid
@@ -278,20 +306,14 @@ func (c *Command) setupAgent(config *Config, logOutput io.Writer, logWriter *log
 	c.Ui.Output("Starting Consul agent RPC...")
 	c.rpcServer = NewAgentRPC(agent, rpcListener, logOutput, logWriter)
 
-	if config.Ports.HTTP > 0 {
-		httpAddr, err := config.ClientListener(config.Addresses.HTTP, config.Ports.HTTP)
-		if err != nil {
-			c.Ui.Error(fmt.Sprintf("Invalid HTTP bind address: %s", err))
-			return err
-		}
-
-		server, err := NewHTTPServer(agent, config.UiDir, config.EnableDebug, logOutput, httpAddr.String())
+	if config.Ports.HTTP > 0 || config.Ports.HTTPS > 0 {
+		servers, err := NewHTTPServers(agent, config, logOutput)
 		if err != nil {
 			agent.Shutdown()
-			c.Ui.Error(fmt.Sprintf("Error starting http server: %s", err))
+			c.Ui.Error(fmt.Sprintf("Error starting http servers: %s", err))
 			return err
 		}
-		c.httpServer = server
+		c.httpServers = servers
 	}
 
 	if config.Ports.DNS > 0 {
@@ -302,7 +324,7 @@ func (c *Command) setupAgent(config *Config, logOutput io.Writer, logWriter *log
 		}
 
 		server, err := NewDNSServer(agent, &config.DNSConfig, logOutput,
-			config.Domain, dnsAddr.String(), config.DNSRecursor)
+			config.Domain, dnsAddr.String(), config.DNSRecursors)
 		if err != nil {
 			agent.Shutdown()
 			c.Ui.Error(fmt.Sprintf("Error starting dns server: %s", err))
@@ -369,6 +391,22 @@ func (c *Command) startupJoin(config *Config) error {
 	return nil
 }
 
+// startupJoinWan is invoked to handle any joins -wan specified to take place at start time
+func (c *Command) startupJoinWan(config *Config) error {
+	if len(config.StartJoinWan) == 0 {
+		return nil
+	}
+
+	c.Ui.Output("Joining -wan cluster...")
+	n, err := c.agent.JoinWAN(config.StartJoinWan)
+	if err != nil {
+		return err
+	}
+
+	c.Ui.Info(fmt.Sprintf("Join -wan completed. Synced with %d initial agents", n))
+	return nil
+}
+
 // retryJoin is used to handle retrying a join until it succeeds or all
 // retries are exhausted.
 func (c *Command) retryJoin(config *Config, errCh chan<- struct{}) {
@@ -400,6 +438,53 @@ func (c *Command) retryJoin(config *Config, errCh chan<- struct{}) {
 	}
 }
 
+// retryJoinWan is used to handle retrying a join -wan until it succeeds or all
+// retries are exhausted.
+func (c *Command) retryJoinWan(config *Config, errCh chan<- struct{}) {
+	if len(config.RetryJoinWan) == 0 {
+		return
+	}
+
+	logger := c.agent.logger
+	logger.Printf("[INFO] agent: Joining WAN cluster...")
+
+	attempt := 0
+	for {
+		n, err := c.agent.JoinWAN(config.RetryJoinWan)
+		if err == nil {
+			logger.Printf("[INFO] agent: Join -wan completed. Synced with %d initial agents", n)
+			return
+		}
+
+		attempt++
+		if config.RetryMaxAttemptsWan > 0 && attempt > config.RetryMaxAttemptsWan {
+			logger.Printf("[ERROR] agent: max join -wan retry exhausted, exiting")
+			close(errCh)
+			return
+		}
+
+		logger.Printf("[WARN] agent: Join -wan failed: %v, retrying in %v", err,
+			config.RetryIntervalWan)
+		time.Sleep(config.RetryIntervalWan)
+	}
+}
+
+// gossipEncrypted determines if the consul instance is using symmetric
+// encryption keys to protect gossip protocol messages.
+func (c *Command) gossipEncrypted() bool {
+	if c.agent.config.EncryptKey != "" {
+		return true
+	}
+
+	server := c.agent.server
+	if server != nil {
+		return server.KeyManagerLAN() != nil || server.KeyManagerWAN() != nil
+	}
+
+	client := c.agent.client
+	return client != nil && client.KeyManagerLAN() != nil
+}
+
 func (c *Command) Run(args []string) int {
 	c.Ui = &cli.PrefixedUi{
 		OutputPrefix: "==> ",
@@ -472,8 +557,9 @@ func (c *Command) Run(args []string) int {
 	if c.rpcServer != nil {
 		defer c.rpcServer.Shutdown()
 	}
-	if c.httpServer != nil {
-		defer c.httpServer.Shutdown()
+
+	for _, server := range c.httpServers {
+		defer server.Shutdown()
 	}
 
 	// Join startup nodes if specified
@@ -482,27 +568,13 @@ func (c *Command) Run(args []string) int {
 		return 1
 	}
 
-	// Register the services
-	for _, service := range config.Services {
-		ns := service.NodeService()
-		chkType := service.CheckType()
-		if err := c.agent.AddService(ns, chkType); err != nil {
-			c.Ui.Error(fmt.Sprintf("Failed to register service '%s': %v", service.Name, err))
-			return 1
-		}
+	// Join startup nodes if specified
+	if err := c.startupJoinWan(config); err != nil {
+		c.Ui.Error(err.Error())
+		return 1
 	}
 
-	// Register the checks
-	for _, check := range config.Checks {
-		health := check.HealthCheck(config.NodeName)
-		chkType := &check.CheckType
-		if err := c.agent.AddCheck(health, chkType); err != nil {
-			c.Ui.Error(fmt.Sprintf("Failed to register check '%s': %v %v", check.Name, err, check))
-			return 1
-		}
-	}
-
-	// Get the new client listener addr
+	// Get the new client http listener addr
 	httpAddr, err := config.ClientListenerAddr(config.Addresses.HTTP, config.Ports.HTTP)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to determine HTTP address: %v", err))
@@ -519,6 +591,14 @@ func (c *Command) Run(args []string) int {
 		}(wp)
 	}
 
+	// Figure out if gossip is encrypted
+	var gossipEncrypted bool
+	if config.Server {
+		gossipEncrypted = c.agent.server.Encrypted()
+	} else {
+		gossipEncrypted = c.agent.client.Encrypted()
+	}
+
 	// Let the agent know we've finished registration
 	c.agent.StartSync()
 
@@ -526,12 +606,12 @@ func (c *Command) Run(args []string) int {
 	c.Ui.Info(fmt.Sprintf("     Node name: '%s'", config.NodeName))
 	c.Ui.Info(fmt.Sprintf("    Datacenter: '%s'", config.Datacenter))
 	c.Ui.Info(fmt.Sprintf("        Server: %v (bootstrap: %v)", config.Server, config.Bootstrap))
-	c.Ui.Info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, DNS: %d, RPC: %d)", config.ClientAddr,
-		config.Ports.HTTP, config.Ports.DNS, config.Ports.RPC))
+	c.Ui.Info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, HTTPS: %d, DNS: %d, RPC: %d)", config.ClientAddr,
+		config.Ports.HTTP, config.Ports.HTTPS, config.Ports.DNS, config.Ports.RPC))
 	c.Ui.Info(fmt.Sprintf("  Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddr,
 		config.Ports.SerfLan, config.Ports.SerfWan))
 	c.Ui.Info(fmt.Sprintf("Gossip encrypt: %v, RPC-TLS: %v, TLS-Incoming: %v",
-		config.EncryptKey != "", config.VerifyOutgoing, config.VerifyIncoming))
+		gossipEncrypted, config.VerifyOutgoing, config.VerifyIncoming))
 
 	// Enable log streaming
 	c.Ui.Info("")
@@ -542,12 +622,16 @@ func (c *Command) Run(args []string) int {
 	errCh := make(chan struct{})
 	go c.retryJoin(config, errCh)
 
+	// Start retry -wan join process
+	errWanCh := make(chan struct{})
+	go c.retryJoinWan(config, errWanCh)
+
 	// Wait for exit
-	return c.handleSignals(config, errCh)
+	return c.handleSignals(config, errCh, errWanCh)
 }
 
 // handleSignals blocks until we get an exit-causing signal
-func (c *Command) handleSignals(config *Config, retryJoin <-chan struct{}) int {
+func (c *Command) handleSignals(config *Config, retryJoin <-chan struct{}, retryJoinWan <-chan struct{}) int {
 	signalCh := make(chan os.Signal, 4)
 	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
 
@@ -563,6 +647,8 @@ WAIT:
 		sig = os.Interrupt
 	case <-retryJoin:
 		return 1
+	case <-retryJoinWan:
+		return 1
 	case <-c.agent.ShutdownCh():
 		// Agent is already shutdown!
 		return 0
@@ -636,34 +722,14 @@ func (c *Command) handleReload(config *Config) *Config {
 	c.agent.PauseSync()
 	defer c.agent.ResumeSync()
 
-	// Deregister the old services
-	for _, service := range config.Services {
-		ns := service.NodeService()
-		c.agent.RemoveService(ns.ID)
+	// Reload services and check definitions
+	if err := c.agent.reloadServices(newConf); err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed reloading services: %s", err))
+		return nil
 	}
-
-	// Deregister the old checks
-	for _, check := range config.Checks {
-		health := check.HealthCheck(config.NodeName)
-		c.agent.RemoveCheck(health.CheckID)
-	}
-
-	// Register the services
-	for _, service := range newConf.Services {
-		ns := service.NodeService()
-		chkType := service.CheckType()
-		if err := c.agent.AddService(ns, chkType); err != nil {
-			c.Ui.Error(fmt.Sprintf("Failed to register service '%s': %v", service.Name, err))
-		}
-	}
-
-	// Register the checks
-	for _, check := range newConf.Checks {
-		health := check.HealthCheck(config.NodeName)
-		chkType := &check.CheckType
-		if err := c.agent.AddCheck(health, chkType); err != nil {
-			c.Ui.Error(fmt.Sprintf("Failed to register check '%s': %v %v", check.Name, err, check))
-		}
+	if err := c.agent.reloadChecks(newConf); err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed reloading checks: %s", err))
+		return nil
 	}
 
 	// Get the new client listener addr
@@ -709,7 +775,7 @@ Options:
   -bind=0.0.0.0            Sets the bind address for cluster communication
   -bootstrap-expect=0      Sets server to expect bootstrap mode.
   -client=127.0.0.1        Sets the address to bind for client access.
-                           This includes RPC, DNS and HTTP
+                           This includes RPC, DNS, HTTP and HTTPS (if configured)
   -config-file=foo         Path to a JSON file to read configuration from.
                            This can be specified multiple times.
   -config-dir=foo          Path to a directory to read configuration files
@@ -721,11 +787,18 @@ Options:
   -encrypt=key             Provides the gossip encryption key
   -join=1.2.3.4            Address of an agent to join at start time.
                            Can be specified multiple times.
+  -join-wan=1.2.3.4        Address of an agent to join -wan at start time.
+                           Can be specified multiple times.
   -retry-join=1.2.3.4      Address of an agent to join at start time with
                            retries enabled. Can be specified multiple times.
   -retry-interval=30s      Time to wait between join attempts.
   -retry-max=0             Maximum number of join attempts. Defaults to 0, which
                            will retry indefinitely.
+  -retry-join-wan=1.2.3.4  Address of an agent to join -wan at start time with
+                           retries enabled. Can be specified multiple times.
+  -retry-interval-wan=30s  Time to wait between join -wan attempts.
+  -retry-max-wan=0         Maximum number of join -wan attempts. Defaults to 0, which
+                           will retry indefinitely.
   -log-level=info          Log level of the agent.
   -node=hostname           Name of this node. Must be unique in the cluster
   -protocol=N              Sets the protocol version. Defaults to latest.

command/agent/command_test.go

@@ -68,11 +68,19 @@ func TestRetryJoin(t *testing.T) {
 		agent.config.BindAddr,
 		agent.config.Ports.SerfLan)
 
+	serfWanAddr := fmt.Sprintf(
+		"%s:%d",
+		agent.config.BindAddr,
+		agent.config.Ports.SerfWan)
+
 	args := []string{
+		"-server",
 		"-data-dir", tmpDir,
 		"-node", fmt.Sprintf(`"%s"`, conf2.NodeName),
 		"-retry-join", serfAddr,
 		"-retry-interval", "1s",
+		"-retry-join-wan", serfWanAddr,
+		"-retry-interval-wan", "1s",
 	}
 
 	go func() {
@@ -87,6 +95,10 @@ func TestRetryJoin(t *testing.T) {
 		if len(mem) != 2 {
 			return false, fmt.Errorf("bad: %#v", mem)
 		}
+		mem = agent.WANMembers()
+		if len(mem) != 2 {
+			return false, fmt.Errorf("bad (wan): %#v", mem)
+		}
 		return true, nil
 	}, func(err error) {
 		t.Fatalf(err.Error())
@@ -121,3 +133,33 @@ func TestRetryJoinFail(t *testing.T) {
 		t.Fatalf("bad: %d", code)
 	}
 }
+
+func TestRetryJoinWanFail(t *testing.T) {
+	conf := nextConfig()
+	tmpDir, err := ioutil.TempDir("", "consul")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	shutdownCh := make(chan struct{})
+	defer close(shutdownCh)
+
+	cmd := &Command{
+		ShutdownCh: shutdownCh,
+		Ui:         new(cli.MockUi),
+	}
+
+	serfAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.Ports.SerfWan)
+
+	args := []string{
+		"-server",
+		"-data-dir", tmpDir,
+		"-retry-join-wan", serfAddr,
+		"-retry-max-wan", "1",
+	}
+
+	if code := cmd.Run(args); code == 0 {
+		t.Fatalf("bad: %d", code)
+	}
+}

command/agent/config.go

@@ -23,6 +23,7 @@ import (
 type PortConfig struct {
 	DNS     int // DNS Query interface
 	HTTP    int // HTTP API
+	HTTPS   int // HTTPS API
 	RPC     int // CLI RPC
 	SerfLan int `mapstructure:"serf_lan"` // LAN gossip (Client + Server)
 	SerfWan int `mapstructure:"serf_wan"` // WAN gossip (Server onlyg)
@@ -33,9 +34,10 @@ type PortConfig struct {
 // for specific services. By default, either ClientAddress
 // or ServerAddress is used.
 type AddressConfig struct {
-	DNS  string // DNS Query interface
-	HTTP string // HTTP API
-	RPC  string // CLI RPC
+	DNS   string // DNS Query interface
+	HTTP  string // HTTP API
+	HTTPS string // HTTPS API
+	RPC   string // CLI RPC
 }
 
 // DNSConfig is used to fine tune the DNS sub-system.
@@ -97,10 +99,15 @@ type Config struct {
 	// DataDir is the directory to store our state in
 	DataDir string `mapstructure:"data_dir"`
 
-	// DNSRecursor can be set to allow the DNS server to recursively
-	// resolve non-consul domains
+	// DNSRecursors can be set to allow the DNS servers to recursively
+	// resolve non-consul domains. It is deprecated, and merges into the
+	// recursors array.
 	DNSRecursor string `mapstructure:"recursor"`
 
+	// DNSRecursors can be set to allow the DNS servers to recursively
+	// resolve non-consul domains
+	DNSRecursors []string `mapstructure:"recursors"`
+
 	// DNS configuration
 	DNSConfig DNSConfig `mapstructure:"dns_config"`
 
@@ -117,7 +124,7 @@ type Config struct {
 	NodeName string `mapstructure:"node_name"`
 
 	// ClientAddr is used to control the address we bind to for
-	// client services (DNS, HTTP, RPC)
+	// client services (DNS, HTTP, HTTPS, RPC)
 	ClientAddr string `mapstructure:"client_addr"`
 
 	// BindAddr is used to control the address we bind to.
@@ -189,6 +196,11 @@ type Config struct {
 	// addresses, then the agent will error and exit.
 	StartJoin []string `mapstructure:"start_join"`
 
+	// StartJoinWan is a list of addresses to attempt to join -wan when the
+	// agent starts. If Serf is unable to communicate with any of these
+	// addresses, then the agent will error and exit.
+	StartJoinWan []string `mapstructure:"start_join_wan"`
+
 	// RetryJoin is a list of addresses to join with retry enabled.
 	RetryJoin []string `mapstructure:"retry_join"`
 
@@ -203,6 +215,20 @@ type Config struct {
 	RetryInterval    time.Duration `mapstructure:"-" json:"-"`
 	RetryIntervalRaw string        `mapstructure:"retry_interval"`
 
+	// RetryJoinWan is a list of addresses to join -wan with retry enabled.
+	RetryJoinWan []string `mapstructure:"retry_join_wan"`
+
+	// RetryMaxAttemptsWan specifies the maximum number of times to retry joining a
+	// -wan host on startup. This is useful for cases where we know the node will be
+	// online eventually.
+	RetryMaxAttemptsWan int `mapstructure:"retry_max_wan"`
+
+	// RetryIntervalWan specifies the amount of time to wait in between join
+	// -wan attempts on agent start. The minimum allowed value is 1 second and
+	// the default is 30s.
+	RetryIntervalWan    time.Duration `mapstructure:"-" json:"-"`
+	RetryIntervalWanRaw string        `mapstructure:"retry_interval_wan"`
+
 	// UiDir is the directory containing the Web UI resources.
 	// If provided, the UI endpoints will be enabled.
 	UiDir string `mapstructure:"ui_dir"`
@@ -327,6 +353,7 @@ func DefaultConfig() *Config {
 		Ports: PortConfig{
 			DNS:     8600,
 			HTTP:    8500,
+			HTTPS:   -1,
 			RPC:     8400,
 			SerfLan: consul.DefaultLANSerfPort,
 			SerfWan: consul.DefaultWANSerfPort,
@@ -343,6 +370,7 @@ func DefaultConfig() *Config {
 		ACLDownPolicy:       "extend-cache",
 		ACLDefaultPolicy:    "allow",
 		RetryInterval:       30 * time.Second,
+		RetryIntervalWan:    30 * time.Second,
 	}
 }
 
@@ -392,16 +420,43 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 
 	// Check the result type
 	if obj, ok := raw.(map[string]interface{}); ok {
-		// Check for a "service" or "check" key, meaning
+		// Check for a "services", "service" or "check" key, meaning
 		// this is actually a definition entry
+		if sub, ok := obj["services"]; ok {
+			if list, ok := sub.([]interface{}); ok {
+				for _, srv := range list {
+					service, err := DecodeServiceDefinition(srv)
+					if err != nil {
+						return nil, err
+					}
+					result.Services = append(result.Services, service)
+				}
+			}
+		}
 		if sub, ok := obj["service"]; ok {
 			service, err := DecodeServiceDefinition(sub)
+			if err != nil {
+				return nil, err
+			}
 			result.Services = append(result.Services, service)
-			return &result, err
-		} else if sub, ok := obj["check"]; ok {
+		}
+		if sub, ok := obj["checks"]; ok {
+			if list, ok := sub.([]interface{}); ok {
+				for _, chk := range list {
+					check, err := DecodeCheckDefinition(chk)
+					if err != nil {
+						return nil, err
+					}
+					result.Checks = append(result.Checks, check)
+				}
+			}
+		}
+		if sub, ok := obj["check"]; ok {
 			check, err := DecodeCheckDefinition(sub)
+			if err != nil {
+				return nil, err
+			}
 			result.Checks = append(result.Checks, check)
-			return &result, err
 		}
 	}
 
@@ -473,6 +528,19 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 		result.RetryInterval = dur
 	}
 
+	if raw := result.RetryIntervalWanRaw; raw != "" {
+		dur, err := time.ParseDuration(raw)
+		if err != nil {
+			return nil, fmt.Errorf("RetryIntervalWan invalid: %v", err)
+		}
+		result.RetryIntervalWan = dur
+	}
+
+	// Merge the single recursor
+	if result.DNSRecursor != "" {
+		result.DNSRecursors = append(result.DNSRecursors, result.DNSRecursor)
+	}
+
 	return &result, nil
 }
 
@@ -491,8 +559,13 @@ func DecodeServiceDefinition(raw interface{}) (*ServiceDefinition, error) {
 		}
 	}
 
-	sub, ok = rawMap["check"]
-	if !ok {
+	for k, v := range rawMap {
+		if strings.ToLower(k) == "check" {
+			sub = v
+			break
+		}
+	}
+	if sub == nil {
 		goto AFTER_FIX
 	}
 	if err := FixupCheckType(sub); err != nil {
@@ -596,9 +669,12 @@ func MergeConfig(a, b *Config) *Config {
 	if b.DataDir != "" {
 		result.DataDir = b.DataDir
 	}
-	if b.DNSRecursor != "" {
-		result.DNSRecursor = b.DNSRecursor
-	}
+
+	// Copy the dns recursors
+	result.DNSRecursors = make([]string, 0, len(a.DNSRecursors)+len(b.DNSRecursors))
+	result.DNSRecursors = append(result.DNSRecursors, a.DNSRecursors...)
+	result.DNSRecursors = append(result.DNSRecursors, b.DNSRecursors...)
+
 	if b.Domain != "" {
 		result.Domain = b.Domain
 	}
@@ -671,6 +747,9 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Ports.HTTP != 0 {
 		result.Ports.HTTP = b.Ports.HTTP
 	}
+	if b.Ports.HTTPS != 0 {
+		result.Ports.HTTPS = b.Ports.HTTPS
+	}
 	if b.Ports.RPC != 0 {
 		result.Ports.RPC = b.Ports.RPC
 	}
@@ -689,6 +768,9 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Addresses.HTTP != "" {
 		result.Addresses.HTTP = b.Addresses.HTTP
 	}
+	if b.Addresses.HTTPS != "" {
+		result.Addresses.HTTPS = b.Addresses.HTTPS
+	}
 	if b.Addresses.RPC != "" {
 		result.Addresses.RPC = b.Addresses.RPC
 	}
@@ -710,6 +792,12 @@ func MergeConfig(a, b *Config) *Config {
 	if b.RetryInterval != 0 {
 		result.RetryInterval = b.RetryInterval
 	}
+	if b.RetryMaxAttemptsWan != 0 {
+		result.RetryMaxAttemptsWan = b.RetryMaxAttemptsWan
+	}
+	if b.RetryIntervalWan != 0 {
+		result.RetryIntervalWan = b.RetryIntervalWan
+	}
 	if b.DNSConfig.NodeTTL != 0 {
 		result.DNSConfig.NodeTTL = b.DNSConfig.NodeTTL
 	}
@@ -776,11 +864,21 @@ func MergeConfig(a, b *Config) *Config {
 	result.StartJoin = append(result.StartJoin, a.StartJoin...)
 	result.StartJoin = append(result.StartJoin, b.StartJoin...)
 
+	// Copy the start join addresses
+	result.StartJoinWan = make([]string, 0, len(a.StartJoinWan)+len(b.StartJoinWan))
+	result.StartJoinWan = append(result.StartJoinWan, a.StartJoinWan...)
+	result.StartJoinWan = append(result.StartJoinWan, b.StartJoinWan...)
+
 	// Copy the retry join addresses
 	result.RetryJoin = make([]string, 0, len(a.RetryJoin)+len(b.RetryJoin))
 	result.RetryJoin = append(result.RetryJoin, a.RetryJoin...)
 	result.RetryJoin = append(result.RetryJoin, b.RetryJoin...)
 
+	// Copy the retry join -wan addresses
+	result.RetryJoinWan = make([]string, 0, len(a.RetryJoinWan)+len(b.RetryJoinWan))
+	result.RetryJoinWan = append(result.RetryJoinWan, a.RetryJoinWan...)
+	result.RetryJoinWan = append(result.RetryJoinWan, b.RetryJoinWan...)
+
 	return &result
 }
 

command/agent/config_test.go

@@ -109,7 +109,7 @@ func TestDecodeConfig(t *testing.T) {
 	}
 
 	// DNS setup
-	input = `{"ports": {"dns": 8500}, "recursor": "8.8.8.8", "domain": "foobar"}`
+	input = `{"ports": {"dns": 8500}, "recursors": ["8.8.8.8","8.8.4.4"], "recursor":"127.0.0.1", "domain": "foobar"}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -119,7 +119,16 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
-	if config.DNSRecursor != "8.8.8.8" {
+	if len(config.DNSRecursors) != 3 {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.DNSRecursors[0] != "8.8.8.8" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.DNSRecursors[1] != "8.8.4.4" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.DNSRecursors[2] != "127.0.0.1" {
 		t.Fatalf("bad: %#v", config)
 	}
 
@@ -128,7 +137,7 @@ func TestDecodeConfig(t *testing.T) {
 	}
 
 	// RPC configs
-	input = `{"ports": {"http": 1234, "rpc": 8100}, "client_addr": "0.0.0.0"}`
+	input = `{"ports": {"http": 1234, "https": 1243, "rpc": 8100}, "client_addr": "0.0.0.0"}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -142,6 +151,10 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	if config.Ports.HTTPS != 1243 {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	if config.Ports.RPC != 8100 {
 		t.Fatalf("bad: %#v", config)
 	}
@@ -265,6 +278,23 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	// Start Join wan
+	input = `{"start_join_wan": ["1.1.1.1", "2.2.2.2"]}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(config.StartJoinWan) != 2 {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.StartJoinWan[0] != "1.1.1.1" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.StartJoinWan[1] != "2.2.2.2" {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	// Retry join
 	input = `{"retry_join": ["1.1.1.1", "2.2.2.2"]}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
@@ -307,6 +337,48 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	// Retry Join wan
+	input = `{"retry_join_wan": ["1.1.1.1", "2.2.2.2"]}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(config.RetryJoinWan) != 2 {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.RetryJoinWan[0] != "1.1.1.1" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.RetryJoinWan[1] != "2.2.2.2" {
+		t.Fatalf("bad: %#v", config)
+	}
+
+	// Retry Interval wan
+	input = `{"retry_interval_wan": "10s"}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if config.RetryIntervalWanRaw != "10s" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.RetryIntervalWan.String() != "10s" {
+		t.Fatalf("bad: %#v", config)
+	}
+
+	// Retry Max wan
+	input = `{"retry_max_wan": 3}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if config.RetryMaxAttemptsWan != 3 {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	// UI Dir
 	input = `{"ui_dir": "/opt/consul-ui"}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
@@ -485,7 +557,7 @@ func TestDecodeConfig(t *testing.T) {
 	}
 
 	// Address overrides
-	input = `{"addresses": {"dns": "0.0.0.0", "http": "127.0.0.1", "rpc": "127.0.0.1"}}`
+	input = `{"addresses": {"dns": "0.0.0.0", "http": "127.0.0.1", "https": "127.0.0.1", "rpc": "127.0.0.1"}}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -497,6 +569,9 @@ func TestDecodeConfig(t *testing.T) {
 	if config.Addresses.HTTP != "127.0.0.1" {
 		t.Fatalf("bad: %#v", config)
 	}
+	if config.Addresses.HTTPS != "127.0.0.1" {
+		t.Fatalf("bad: %#v", config)
+	}
 	if config.Addresses.RPC != "127.0.0.1" {
 		t.Fatalf("bad: %#v", config)
 	}
@@ -516,6 +591,195 @@ func TestDecodeConfig(t *testing.T) {
 	}
 }
 
+func TestDecodeConfig_Services(t *testing.T) {
+	input := `{
+		"services": [
+			{
+				"id": "red0",
+				"name": "redis",
+				"tags": [
+					"master"
+				],
+				"port": 6000,
+				"check": {
+					"script": "/bin/check_redis -p 6000",
+					"interval": "5s",
+					"ttl": "20s"
+				}
+			},
+			{
+				"id": "red1",
+				"name": "redis",
+				"tags": [
+					"delayed",
+					"slave"
+				],
+				"port": 7000,
+				"check": {
+					"script": "/bin/check_redis -p 7000",
+					"interval": "30s",
+					"ttl": "60s"
+				}
+			}
+		]
+	}`
+
+	config, err := DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		Services: []*ServiceDefinition{
+			&ServiceDefinition{
+				Check: CheckType{
+					Interval: 5 * time.Second,
+					Script:   "/bin/check_redis -p 6000",
+					TTL:      20 * time.Second,
+				},
+				ID:   "red0",
+				Name: "redis",
+				Tags: []string{
+					"master",
+				},
+				Port: 6000,
+			},
+			&ServiceDefinition{
+				Check: CheckType{
+					Interval: 30 * time.Second,
+					Script:   "/bin/check_redis -p 7000",
+					TTL:      60 * time.Second,
+				},
+				ID:   "red1",
+				Name: "redis",
+				Tags: []string{
+					"delayed",
+					"slave",
+				},
+				Port: 7000,
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(config, expected) {
+		t.Fatalf("bad: %#v", config)
+	}
+}
+
+func TestDecodeConfig_Checks(t *testing.T) {
+	input := `{
+		"checks": [
+			{
+				"id": "chk1",
+				"name": "mem",
+				"script": "/bin/check_mem",
+				"interval": "5s"
+			},
+			{
+				"id": "chk2",
+				"name": "cpu",
+				"script": "/bin/check_cpu",
+				"interval": "10s"
+			}
+		]
+	}`
+
+	config, err := DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		Checks: []*CheckDefinition{
+			&CheckDefinition{
+				ID:   "chk1",
+				Name: "mem",
+				CheckType: CheckType{
+					Script:   "/bin/check_mem",
+					Interval: 5 * time.Second,
+				},
+			},
+			&CheckDefinition{
+				ID:   "chk2",
+				Name: "cpu",
+				CheckType: CheckType{
+					Script:   "/bin/check_cpu",
+					Interval: 10 * time.Second,
+				},
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(config, expected) {
+		t.Fatalf("bad: %#v", config)
+	}
+}
+
+func TestDecodeConfig_Multiples(t *testing.T) {
+	input := `{
+		"services": [
+			{
+				"id": "red0",
+				"name": "redis",
+				"tags": [
+					"master"
+				],
+				"port": 6000,
+				"check": {
+					"script": "/bin/check_redis -p 6000",
+					"interval": "5s",
+					"ttl": "20s"
+				}
+			}
+		],
+		"checks": [
+			{
+				"id": "chk1",
+				"name": "mem",
+				"script": "/bin/check_mem",
+				"interval": "10s"
+			}
+		]
+	}`
+
+	config, err := DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		Services: []*ServiceDefinition{
+			&ServiceDefinition{
+				Check: CheckType{
+					Interval: 5 * time.Second,
+					Script:   "/bin/check_redis -p 6000",
+					TTL:      20 * time.Second,
+				},
+				ID:   "red0",
+				Name: "redis",
+				Tags: []string{
+					"master",
+				},
+				Port: 6000,
+			},
+		},
+		Checks: []*CheckDefinition{
+			&CheckDefinition{
+				ID:   "chk1",
+				Name: "mem",
+				CheckType: CheckType{
+					Script:   "/bin/check_mem",
+					Interval: 10 * time.Second,
+				},
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(config, expected) {
+		t.Fatalf("bad: %#v", config)
+	}
+}
+
 func TestDecodeConfig_Service(t *testing.T) {
 	// Basics
 	input := `{"service": {"id": "red1", "name": "redis", "tags": ["master"], "port":8000, "check": {"script": "/bin/check_redis", "interval": "10s", "ttl": "15s" }}}`
@@ -602,7 +866,6 @@ func TestMergeConfig(t *testing.T) {
 		BootstrapExpect:        0,
 		Datacenter:             "dc1",
 		DataDir:                "/tmp/foo",
-		DNSRecursor:            "127.0.0.1:1001",
 		Domain:                 "basic",
 		LogLevel:               "debug",
 		NodeName:               "foo",
@@ -615,6 +878,7 @@ func TestMergeConfig(t *testing.T) {
 		EnableDebug:            false,
 		CheckUpdateIntervalRaw: "8m",
 		RetryIntervalRaw:       "10s",
+		RetryIntervalWanRaw:    "10s",
 	}
 
 	b := &Config{
@@ -622,7 +886,7 @@ func TestMergeConfig(t *testing.T) {
 		BootstrapExpect: 3,
 		Datacenter:      "dc2",
 		DataDir:         "/tmp/bar",
-		DNSRecursor:     "127.0.0.2:1001",
+		DNSRecursors:    []string{"127.0.0.2:1001"},
 		DNSConfig: DNSConfig{
 			NodeTTL: 10 * time.Second,
 			ServiceTTL: map[string]time.Duration{
@@ -645,11 +909,13 @@ func TestMergeConfig(t *testing.T) {
 			SerfLan: 4,
 			SerfWan: 5,
 			Server:  6,
+			HTTPS:   7,
 		},
 		Addresses: AddressConfig{
-			DNS:  "127.0.0.1",
-			HTTP: "127.0.0.2",
-			RPC:  "127.0.0.3",
+			DNS:   "127.0.0.1",
+			HTTP:  "127.0.0.2",
+			RPC:   "127.0.0.3",
+			HTTPS: "127.0.0.4",
 		},
 		Server:                 true,
 		LeaveOnTerm:            true,
@@ -663,12 +929,16 @@ func TestMergeConfig(t *testing.T) {
 		Checks:                 []*CheckDefinition{nil},
 		Services:               []*ServiceDefinition{nil},
 		StartJoin:              []string{"1.1.1.1"},
+		StartJoinWan:           []string{"1.1.1.1"},
 		UiDir:                  "/opt/consul-ui",
 		EnableSyslog:           true,
 		RejoinAfterLeave:       true,
 		RetryJoin:              []string{"1.1.1.1"},
 		RetryIntervalRaw:       "10s",
 		RetryInterval:          10 * time.Second,
+		RetryJoinWan:           []string{"1.1.1.1"},
+		RetryIntervalWanRaw:    "10s",
+		RetryIntervalWan:       10 * time.Second,
 		CheckUpdateInterval:    8 * time.Minute,
 		CheckUpdateIntervalRaw: "8m",
 		ACLToken:               "1234",
@@ -695,7 +965,7 @@ func TestMergeConfig(t *testing.T) {
 	c := MergeConfig(a, b)
 
 	if !reflect.DeepEqual(c, b) {
-		t.Fatalf("should be equal %v %v", c, b)
+		t.Fatalf("should be equal %#v %#v", c, b)
 	}
 }
 

command/agent/dns.go

@@ -2,14 +2,15 @@ package agent
 
 import (
 	"fmt"
-	"github.com/hashicorp/consul/consul/structs"
-	"github.com/miekg/dns"
 	"io"
 	"log"
 	"math/rand"
 	"net"
 	"strings"
 	"time"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/miekg/dns"
 )
 
 const (
@@ -28,12 +29,12 @@ type DNSServer struct {
 	dnsServer    *dns.Server
 	dnsServerTCP *dns.Server
 	domain       string
-	recursor     string
+	recursors    []string
 	logger       *log.Logger
 }
 
 // NewDNSServer starts a new DNS server to provide an agent interface
-func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain, bind, recursor string) (*DNSServer, error) {
+func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain string, bind string, recursors []string) (*DNSServer, error) {
 	// Make sure domain is FQDN
 	domain = dns.Fqdn(domain)
 
@@ -61,21 +62,30 @@ func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain,
 		dnsServer:    server,
 		dnsServerTCP: serverTCP,
 		domain:       domain,
-		recursor:     recursor,
+		recursors:    recursors,
 		logger:       log.New(logOutput, "", log.LstdFlags),
 	}
 
+	// Register mux handler, for reverse lookup
+	mux.HandleFunc("arpa.", srv.handlePtr)
+
 	// Register mux handlers, always handle "consul."
 	mux.HandleFunc(domain, srv.handleQuery)
 	if domain != consulDomain {
 		mux.HandleFunc(consulDomain, srv.handleTest)
 	}
-	if recursor != "" {
-		recursor, err := recursorAddr(recursor)
-		if err != nil {
-			return nil, fmt.Errorf("Invalid recursor address: %v", err)
+	if len(recursors) > 0 {
+		validatedRecursors := make([]string, len(recursors))
+
+		for idx, recursor := range recursors {
+			recursor, err := recursorAddr(recursor)
+			if err != nil {
+				return nil, fmt.Errorf("Invalid recursor address: %v", err)
+			}
+			validatedRecursors[idx] = recursor
 		}
-		srv.recursor = recursor
+
+		srv.recursors = validatedRecursors
 		mux.HandleFunc(".", srv.handleRecurse)
 	}
 
@@ -129,7 +139,6 @@ func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain,
 	case <-time.After(time.Second):
 		return srv, fmt.Errorf("timeout setting up DNS server")
 	}
-	return srv, nil
 }
 
 // recursorAddr is used to add a port to the recursor if omitted.
@@ -155,6 +164,57 @@ START:
 	return addr.String(), nil
 }
 
+// handlePtr is used to handle "reverse" DNS queries
+func (d *DNSServer) handlePtr(resp dns.ResponseWriter, req *dns.Msg) {
+	q := req.Question[0]
+	defer func(s time.Time) {
+		d.logger.Printf("[DEBUG] dns: request for %v (%v)", q, time.Now().Sub(s))
+	}(time.Now())
+
+	// Setup the message response
+	m := new(dns.Msg)
+	m.SetReply(req)
+	m.Authoritative = true
+	m.RecursionAvailable = (len(d.recursors) > 0)
+
+	// Only add the SOA if requested
+	if req.Question[0].Qtype == dns.TypeSOA {
+		d.addSOA(d.domain, m)
+	}
+
+	datacenter := d.agent.config.Datacenter
+
+	// Get the QName without the domain suffix
+	qName := strings.ToLower(dns.Fqdn(req.Question[0].Name))
+
+	args := structs.DCSpecificRequest{
+		Datacenter:   datacenter,
+		QueryOptions: structs.QueryOptions{AllowStale: d.config.AllowStale},
+	}
+	var out structs.IndexedNodes
+
+	// TODO: Replace ListNodes with an internal RPC that can do the filter
+	// server side to avoid transferring the entire node list.
+	if err := d.agent.RPC("Catalog.ListNodes", &args, &out); err == nil {
+		for _, n := range out.Nodes {
+			arpa, _ := dns.ReverseAddr(n.Address)
+			if arpa == qName {
+				ptr := &dns.PTR{
+					Hdr: dns.RR_Header{Name: q.Name, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 0},
+					Ptr: fmt.Sprintf("%s.node.%s.consul.", n.Node, datacenter),
+				}
+				m.Answer = append(m.Answer, ptr)
+				break
+			}
+		}
+	}
+
+	// Write out the complete response
+	if err := resp.WriteMsg(m); err != nil {
+		d.logger.Printf("[WARN] dns: failed to respond: %v", err)
+	}
+}
+
 // handleQUery is used to handle DNS queries in the configured domain
 func (d *DNSServer) handleQuery(resp dns.ResponseWriter, req *dns.Msg) {
 	q := req.Question[0]
@@ -178,7 +238,7 @@ func (d *DNSServer) handleQuery(resp dns.ResponseWriter, req *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(req)
 	m.Authoritative = true
-	m.RecursionAvailable = (d.recursor != "")
+	m.RecursionAvailable = (len(d.recursors) > 0)
 
 	// Only add the SOA if requested
 	if req.Question[0].Qtype == dns.TypeSOA {
@@ -214,7 +274,7 @@ func (d *DNSServer) handleTest(resp dns.ResponseWriter, req *dns.Msg) {
 	m.Authoritative = true
 	m.RecursionAvailable = true
 	header := dns.RR_Header{Name: q.Name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: 0}
-	txt := &dns.TXT{header, []string{"ok"}}
+	txt := &dns.TXT{Hdr: header, Txt: []string{"ok"}}
 	m.Answer = append(m.Answer, txt)
 	d.addSOA(consulDomain, m)
 	if err := resp.WriteMsg(m); err != nil {
@@ -587,30 +647,35 @@ func (d *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
 
 	// Recursively resolve
 	c := &dns.Client{Net: network}
-	r, rtt, err := c.Exchange(req, d.recursor)
-
-	// On failure, return a SERVFAIL message
-	if err != nil {
+	var r *dns.Msg
+	var rtt time.Duration
+	var err error
+	for _, recursor := range d.recursors {
+		r, rtt, err = c.Exchange(req, recursor)
+		if err == nil {
+			// Forward the response
+			d.logger.Printf("[DEBUG] dns: recurse RTT for %v (%v)", q, rtt)
+			if err := resp.WriteMsg(r); err != nil {
+				d.logger.Printf("[WARN] dns: failed to respond: %v", err)
+			}
+			return
+		}
 		d.logger.Printf("[ERR] dns: recurse failed: %v", err)
-		m := &dns.Msg{}
-		m.SetReply(req)
-		m.RecursionAvailable = true
-		m.SetRcode(req, dns.RcodeServerFailure)
-		resp.WriteMsg(m)
-		return
 	}
-	d.logger.Printf("[DEBUG] dns: recurse RTT for %v (%v)", q, rtt)
 
-	// Forward the response
-	if err := resp.WriteMsg(r); err != nil {
-		d.logger.Printf("[WARN] dns: failed to respond: %v", err)
-	}
+	// If all resolvers fail, return a SERVFAIL message
+	d.logger.Printf("[ERR] dns: all resolvers failed for %v", q)
+	m := &dns.Msg{}
+	m.SetReply(req)
+	m.RecursionAvailable = true
+	m.SetRcode(req, dns.RcodeServerFailure)
+	resp.WriteMsg(m)
 }
 
 // resolveCNAME is used to recursively resolve CNAME records
 func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 	// Do nothing if we don't have a recursor
-	if d.recursor == "" {
+	if len(d.recursors) == 0 {
 		return nil
 	}
 
@@ -620,13 +685,17 @@ func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 
 	// Make a DNS lookup request
 	c := &dns.Client{Net: "udp"}
-	r, rtt, err := c.Exchange(m, d.recursor)
-	if err != nil {
-		d.logger.Printf("[ERR] dns: cname recurse failed: %v", err)
-		return nil
+	var r *dns.Msg
+	var rtt time.Duration
+	var err error
+	for _, recursor := range d.recursors {
+		r, rtt, err = c.Exchange(m, recursor)
+		if err == nil {
+			d.logger.Printf("[DEBUG] dns: cname recurse RTT for %v (%v)", name, rtt)
+			return r.Answer
+		}
+		d.logger.Printf("[ERR] dns: cname recurse failed for %v: %v", name, err)
 	}
-	d.logger.Printf("[DEBUG] dns: cname recurse RTT for %v (%v)", name, rtt)
-
-	// Return all the answers
-	return r.Answer
+	d.logger.Printf("[ERR] dns: all resolvers failed for %v", name)
+	return nil
 }

command/agent/dns_test.go

@@ -22,7 +22,7 @@ func makeDNSServerConfig(t *testing.T, config *DNSConfig) (string, *DNSServer) {
 	addr, _ := conf.ClientListener(conf.Addresses.DNS, conf.Ports.DNS)
 	dir, agent := makeAgent(t, conf)
 	server, err := NewDNSServer(agent, config, agent.logOutput,
-		conf.Domain, addr.String(), "8.8.8.8:53")
+		conf.Domain, addr.String(), []string{"8.8.8.8:53"})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -304,6 +304,89 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 	}
 }
 
+func TestDNS_ReverseLookup(t *testing.T) {
+	dir, srv := makeDNSServer(t)
+	defer os.RemoveAll(dir)
+	defer srv.agent.Shutdown()
+
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
+
+	// Register node
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+		Node:       "foo2",
+		Address:    "127.0.0.2",
+	}
+
+	var out struct{}
+	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	m := new(dns.Msg)
+	m.SetQuestion("2.0.0.127.in-addr.arpa.", dns.TypeANY)
+
+	c := new(dns.Client)
+	addr, _ := srv.agent.config.ClientListener("", srv.agent.config.Ports.DNS)
+	in, _, err := c.Exchange(m, addr.String())
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if len(in.Answer) != 1 {
+		t.Fatalf("Bad: %#v", in)
+	}
+
+	ptrRec, ok := in.Answer[0].(*dns.PTR)
+	if !ok {
+		t.Fatalf("Bad: %#v", in.Answer[0])
+	}
+	if ptrRec.Ptr != "foo2.node.dc1.consul." {
+		t.Fatalf("Bad: %#v", ptrRec)
+	}
+}
+
+func TestDNS_ReverseLookup_IPV6(t *testing.T) {
+	dir, srv := makeDNSServer(t)
+	defer os.RemoveAll(dir)
+	defer srv.agent.Shutdown()
+
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
+
+	// Register node
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+		Node:       "bar",
+		Address:    "::4242:4242",
+	}
+
+	var out struct{}
+	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	m := new(dns.Msg)
+	m.SetQuestion("2.4.2.4.2.4.2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa.", dns.TypeANY)
+
+	c := new(dns.Client)
+	addr, _ := srv.agent.config.ClientListener("", srv.agent.config.Ports.DNS)
+	in, _, err := c.Exchange(m, addr.String())
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if len(in.Answer) != 1 {
+		t.Fatalf("Bad: %#v", in)
+	}
+
+	ptrRec, ok := in.Answer[0].(*dns.PTR)
+	if !ok {
+		t.Fatalf("Bad: %#v", in.Answer[0])
+	}
+	if ptrRec.Ptr != "bar.node.dc1.consul." {
+		t.Fatalf("Bad: %#v", ptrRec)
+	}
+}
 func TestDNS_ServiceLookup(t *testing.T) {
 	dir, srv := makeDNSServer(t)
 	defer os.RemoveAll(dir)

command/agent/event_endpoint.go

@@ -118,16 +118,12 @@ RUN_QUERY:
 
 	// Filter the events if necessary
 	if nameFilter != "" {
-		n := len(events)
-		for i := 0; i < n; i++ {
-			if events[i].Name == nameFilter {
-				continue
+		for i := 0; i < len(events); i++ {
+			if events[i].Name != nameFilter {
+				events = append(events[:i], events[i+1:]...)
+				i--
 			}
-			events[i], events[n-1] = events[n-1], nil
-			i--
-			n--
 		}
-		events = events[:n]
 	}
 
 	// Determine the index

command/agent/event_endpoint_test.go

@@ -190,6 +190,50 @@ func TestEventList_Blocking(t *testing.T) {
 	})
 }
 
+func TestEventList_EventBufOrder(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		// Fire some events in a non-sequential order
+		expected := &UserEvent{Name: "foo"}
+
+		for _, e := range []*UserEvent{
+			&UserEvent{Name: "foo"},
+			&UserEvent{Name: "bar"},
+			&UserEvent{Name: "foo"},
+			expected,
+			&UserEvent{Name: "bar"},
+		} {
+			if err := srv.agent.UserEvent("", e); err != nil {
+				t.Fatalf("err: %v", err)
+			}
+		}
+
+		// Test that the event order is preserved when name
+		// filtering on a list of > 1 matching event.
+		testutil.WaitForResult(func() (bool, error) {
+			url := "/v1/event/list?name=foo"
+			req, err := http.NewRequest("GET", url, nil)
+			if err != nil {
+				return false, err
+			}
+			resp := httptest.NewRecorder()
+			obj, err := srv.EventList(resp, req)
+			if err != nil {
+				return false, err
+			}
+			list, ok := obj.([]*UserEvent)
+			if !ok {
+				return false, fmt.Errorf("bad: %#v", obj)
+			}
+			if len(list) != 3 || list[2].ID != expected.ID {
+				return false, fmt.Errorf("bad: %#v", list)
+			}
+			return true, nil
+		}, func(err error) {
+			t.Fatalf("err: %v", err)
+		})
+	})
+}
+
 func TestUUIDToUint64(t *testing.T) {
 	inp := "cb9a81ad-fff6-52ac-92a7-5f70687805ec"
 

command/agent/http.go

@@ -1,7 +1,9 @@
 package agent
 
 import (
+	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net"
@@ -12,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/tlsutil"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -23,38 +26,123 @@ type HTTPServer struct {
 	listener net.Listener
 	logger   *log.Logger
 	uiDir    string
+	addr     string
 }
 
-// NewHTTPServer starts a new HTTP server to provide an interface to
+// NewHTTPServers starts new HTTP servers to provide an interface to
 // the agent.
-func NewHTTPServer(agent *Agent, uiDir string, enableDebug bool, logOutput io.Writer, bind string) (*HTTPServer, error) {
-	// Create the mux
-	mux := http.NewServeMux()
+func NewHTTPServers(agent *Agent, config *Config, logOutput io.Writer) ([]*HTTPServer, error) {
+	var tlsConfig *tls.Config
+	var list net.Listener
+	var httpAddr *net.TCPAddr
+	var err error
+	var servers []*HTTPServer
 
-	// Create listener
-	list, err := net.Listen("tcp", bind)
+	if config.Ports.HTTPS > 0 {
+		httpAddr, err = config.ClientListener(config.Addresses.HTTPS, config.Ports.HTTPS)
+		if err != nil {
+			return nil, err
+		}
+
+		tlsConf := &tlsutil.Config{
+			VerifyIncoming: config.VerifyIncoming,
+			VerifyOutgoing: config.VerifyOutgoing,
+			CAFile:         config.CAFile,
+			CertFile:       config.CertFile,
+			KeyFile:        config.KeyFile,
+			NodeName:       config.NodeName,
+			ServerName:     config.ServerName}
+
+		tlsConfig, err = tlsConf.IncomingTLSConfig()
+		if err != nil {
+			return nil, err
+		}
+
+		ln, err := net.Listen("tcp", httpAddr.String())
+		if err != nil {
+			return nil, err
+		}
+
+		list = tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, tlsConfig)
+
+		// Create the mux
+		mux := http.NewServeMux()
+
+		// Create the server
+		srv := &HTTPServer{
+			agent:    agent,
+			mux:      mux,
+			listener: list,
+			logger:   log.New(logOutput, "", log.LstdFlags),
+			uiDir:    config.UiDir,
+			addr:     httpAddr.String(),
+		}
+		srv.registerHandlers(config.EnableDebug)
+
+		// Start the server
+		go http.Serve(list, mux)
+		servers = append(servers, srv)
+	}
+
+	if config.Ports.HTTP > 0 {
+		httpAddr, err = config.ClientListener(config.Addresses.HTTP, config.Ports.HTTP)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get ClientListener address:port: %v", err)
+		}
+
+		// Create non-TLS listener
+		ln, err := net.Listen("tcp", httpAddr.String())
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get Listen on %s: %v", httpAddr.String(), err)
+		}
+
+		list = tcpKeepAliveListener{ln.(*net.TCPListener)}
+
+		// Create the mux
+		mux := http.NewServeMux()
+
+		// Create the server
+		srv := &HTTPServer{
+			agent:    agent,
+			mux:      mux,
+			listener: list,
+			logger:   log.New(logOutput, "", log.LstdFlags),
+			uiDir:    config.UiDir,
+			addr:     httpAddr.String(),
+		}
+		srv.registerHandlers(config.EnableDebug)
+
+		// Start the server
+		go http.Serve(list, mux)
+		servers = append(servers, srv)
+	}
+
+	return servers, nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by NewHttpServer so
+// dead TCP connections eventually go away.
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
 	if err != nil {
-		return nil, err
+		return
 	}
-
-	// Create the server
-	srv := &HTTPServer{
-		agent:    agent,
-		mux:      mux,
-		listener: list,
-		logger:   log.New(logOutput, "", log.LstdFlags),
-		uiDir:    uiDir,
-	}
-	srv.registerHandlers(enableDebug)
-
-	// Start the server
-	go http.Serve(list, mux)
-	return srv, nil
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(30 * time.Second)
+	return tc, nil
 }
 
 // Shutdown is used to shutdown the HTTP server
 func (s *HTTPServer) Shutdown() {
-	s.listener.Close()
+	if s != nil {
+		s.logger.Printf("[DEBUG] http: Shutting down http server(%v)", s.addr)
+		s.listener.Close()
+	}
 }
 
 // registerHandlers is used to attach our handlers to the mux
@@ -100,6 +188,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 
 	s.mux.HandleFunc("/v1/session/create", s.wrap(s.SessionCreate))
 	s.mux.HandleFunc("/v1/session/destroy/", s.wrap(s.SessionDestroy))
+	s.mux.HandleFunc("/v1/session/renew/", s.wrap(s.SessionRenew))
 	s.mux.HandleFunc("/v1/session/info/", s.wrap(s.SessionGet))
 	s.mux.HandleFunc("/v1/session/node/", s.wrap(s.SessionsForNode))
 	s.mux.HandleFunc("/v1/session/list", s.wrap(s.SessionList))

command/agent/http_test.go

@@ -25,12 +25,15 @@ func makeHTTPServer(t *testing.T) (string, *HTTPServer) {
 	if err := os.Mkdir(uiDir, 755); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	addr, _ := agent.config.ClientListener("", agent.config.Ports.HTTP)
-	server, err := NewHTTPServer(agent, uiDir, true, agent.logOutput, addr.String())
+	conf.UiDir = uiDir
+	servers, err := NewHTTPServers(agent, conf, agent.logOutput)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	return dir, server
+	if len(servers) == 0 {
+		t.Fatalf(fmt.Sprintf("Failed to make HTTP server"))
+	}
+	return dir, servers[0]
 }
 
 func encodeReq(obj interface{}) io.ReadCloser {

command/agent/keyring.go

@@ -0,0 +1,145 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/serf"
+)
+
+const (
+	serfLANKeyring = "serf/local.keyring"
+	serfWANKeyring = "serf/remote.keyring"
+)
+
+// initKeyring will create a keyring file at a given path.
+func initKeyring(path, key string) error {
+	var keys []string
+
+	if _, err := base64.StdEncoding.DecodeString(key); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	}
+
+	// Just exit if the file already exists.
+	if _, err := os.Stat(path); err == nil {
+		return nil
+	}
+
+	keys = append(keys, key)
+	keyringBytes, err := json.Marshal(keys)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return err
+	}
+
+	fh, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	if _, err := fh.Write(keyringBytes); err != nil {
+		os.Remove(path)
+		return err
+	}
+
+	return nil
+}
+
+// loadKeyringFile will load a gossip encryption keyring out of a file. The file
+// must be in JSON format and contain a list of encryption key strings.
+func loadKeyringFile(c *serf.Config) error {
+	if c.KeyringFile == "" {
+		return nil
+	}
+
+	if _, err := os.Stat(c.KeyringFile); err != nil {
+		return err
+	}
+
+	// Read in the keyring file data
+	keyringData, err := ioutil.ReadFile(c.KeyringFile)
+	if err != nil {
+		return err
+	}
+
+	// Decode keyring JSON
+	keys := make([]string, 0)
+	if err := json.Unmarshal(keyringData, &keys); err != nil {
+		return err
+	}
+
+	// Decode base64 values
+	keysDecoded := make([][]byte, len(keys))
+	for i, key := range keys {
+		keyBytes, err := base64.StdEncoding.DecodeString(key)
+		if err != nil {
+			return err
+		}
+		keysDecoded[i] = keyBytes
+	}
+
+	// Guard against empty keyring
+	if len(keysDecoded) == 0 {
+		return fmt.Errorf("no keys present in keyring file: %s", c.KeyringFile)
+	}
+
+	// Create the keyring
+	keyring, err := memberlist.NewKeyring(keysDecoded, keysDecoded[0])
+	if err != nil {
+		return err
+	}
+
+	c.MemberlistConfig.Keyring = keyring
+
+	// Success!
+	return nil
+}
+
+// keyringProcess is used to abstract away the semantic similarities in
+// performing various operations on the encryption keyring.
+func (a *Agent) keyringProcess(args *structs.KeyringRequest) (*structs.KeyringResponses, error) {
+	var reply structs.KeyringResponses
+	if a.server == nil {
+		return nil, fmt.Errorf("keyring operations must run against a server node")
+	}
+	if err := a.RPC("Internal.KeyringOperation", args, &reply); err != nil {
+		return &reply, err
+	}
+
+	return &reply, nil
+}
+
+// ListKeys lists out all keys installed on the collective Consul cluster. This
+// includes both servers and clients in all DC's.
+func (a *Agent) ListKeys() (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Operation: structs.KeyringList}
+	return a.keyringProcess(&args)
+}
+
+// InstallKey installs a new gossip encryption key
+func (a *Agent) InstallKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringInstall}
+	return a.keyringProcess(&args)
+}
+
+// UseKey changes the primary encryption key used to encrypt messages
+func (a *Agent) UseKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringUse}
+	return a.keyringProcess(&args)
+}
+
+// RemoveKey will remove a gossip encryption key from the keyring
+func (a *Agent) RemoveKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringRemove}
+	return a.keyringProcess(&args)
+}

command/agent/keyring_test.go

@@ -0,0 +1,115 @@
+package agent
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAgent_LoadKeyrings(t *testing.T) {
+	key := "tbLJg26ZJyJ9pK3qhc9jig=="
+
+	// Should be no configured keyring file by default
+	conf1 := nextConfig()
+	dir1, agent1 := makeAgent(t, conf1)
+	defer os.RemoveAll(dir1)
+	defer agent1.Shutdown()
+
+	c := agent1.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfLANConfig.KeyringFile)
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfLANConfig.KeyringFile)
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+
+	// Server should auto-load LAN and WAN keyring files
+	conf2 := nextConfig()
+	dir2, agent2 := makeAgentKeyring(t, conf2, key)
+	defer os.RemoveAll(dir2)
+	defer agent2.Shutdown()
+
+	c = agent2.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+
+	// Client should auto-load only the LAN keyring file
+	conf3 := nextConfig()
+	conf3.Server = false
+	dir3, agent3 := makeAgentKeyring(t, conf3, key)
+	defer os.RemoveAll(dir3)
+	defer agent3.Shutdown()
+
+	c = agent3.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfWANConfig.KeyringFile)
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+}
+
+func TestAgent_InitKeyring(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
+	expected := fmt.Sprintf(`["%s"]`, key1)
+
+	dir, err := ioutil.TempDir("", "consul")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "keyring")
+
+	// First initialize the keyring
+	if err := initKeyring(file, key1); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+
+	// Try initializing again with a different key
+	if err := initKeyring(file, key2); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Content should still be the same
+	content, err = ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+}

command/agent/kvs_endpoint.go

@@ -3,11 +3,12 @@ package agent
 import (
 	"bytes"
 	"fmt"
-	"github.com/hashicorp/consul/consul/structs"
 	"io"
 	"net/http"
 	"strconv"
 	"strings"
+
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 const (
@@ -50,7 +51,6 @@ func (s *HTTPServer) KVSEndpoint(resp http.ResponseWriter, req *http.Request) (i
 		resp.WriteHeader(405)
 		return nil, nil
 	}
-	return nil, nil
 }
 
 // KVSGet handles a GET request

command/agent/local.go

@@ -1,18 +1,23 @@
 package agent
 
 import (
-	"github.com/hashicorp/consul/consul"
-	"github.com/hashicorp/consul/consul/structs"
 	"log"
 	"reflect"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/hashicorp/consul/consul"
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 const (
 	syncStaggerIntv = 3 * time.Second
 	syncRetryIntv   = 15 * time.Second
+
+	// permissionDenied is returned when an ACL based rejection happens
+	permissionDenied = "Permission denied"
 )
 
 // syncStatus is used to represent the difference between
@@ -47,7 +52,7 @@ type localState struct {
 	checks      map[string]*structs.HealthCheck
 	checkStatus map[string]syncStatus
 
-	// Used to track checks that are being defered
+	// Used to track checks that are being deferred
 	deferCheck map[string]*time.Timer
 
 	// consulCh is used to inform of a change to the known
@@ -96,13 +101,13 @@ func (l *localState) ConsulServerUp() {
 	}
 }
 
-// Pause is used to pause state syncronization, this can be
+// Pause is used to pause state synchronization, this can be
 // used to make batch changes
 func (l *localState) Pause() {
 	atomic.StoreInt32(&l.paused, 1)
 }
 
-// Resume is used to resume state syncronization
+// Resume is used to resume state synchronization
 func (l *localState) Resume() {
 	atomic.StoreInt32(&l.paused, 0)
 	l.changeMade()
@@ -292,8 +297,9 @@ SYNC:
 // the local syncStatus as appropriate
 func (l *localState) setSyncState() error {
 	req := structs.NodeSpecificRequest{
-		Datacenter: l.config.Datacenter,
-		Node:       l.config.NodeName,
+		Datacenter:   l.config.Datacenter,
+		Node:         l.config.NodeName,
+		QueryOptions: structs.QueryOptions{Token: l.config.ACLToken},
 	}
 	var out1 structs.IndexedNodeServices
 	var out2 structs.IndexedHealthChecks
@@ -384,7 +390,7 @@ func (l *localState) syncChanges() error {
 				return err
 			}
 		} else if !status.inSync {
-			// Cancel a defered sync
+			// Cancel a deferred sync
 			if timer := l.deferCheck[id]; timer != nil {
 				timer.Stop()
 				delete(l.deferCheck, id)
@@ -403,9 +409,10 @@ func (l *localState) syncChanges() error {
 // deleteService is used to delete a service from the server
 func (l *localState) deleteService(id string) error {
 	req := structs.DeregisterRequest{
-		Datacenter: l.config.Datacenter,
-		Node:       l.config.NodeName,
-		ServiceID:  id,
+		Datacenter:   l.config.Datacenter,
+		Node:         l.config.NodeName,
+		ServiceID:    id,
+		WriteRequest: structs.WriteRequest{Token: l.config.ACLToken},
 	}
 	var out struct{}
 	err := l.iface.RPC("Catalog.Deregister", &req, &out)
@@ -419,9 +426,10 @@ func (l *localState) deleteService(id string) error {
 // deleteCheck is used to delete a service from the server
 func (l *localState) deleteCheck(id string) error {
 	req := structs.DeregisterRequest{
-		Datacenter: l.config.Datacenter,
-		Node:       l.config.NodeName,
-		CheckID:    id,
+		Datacenter:   l.config.Datacenter,
+		Node:         l.config.NodeName,
+		CheckID:      id,
+		WriteRequest: structs.WriteRequest{Token: l.config.ACLToken},
 	}
 	var out struct{}
 	err := l.iface.RPC("Catalog.Deregister", &req, &out)
@@ -435,16 +443,21 @@ func (l *localState) deleteCheck(id string) error {
 // syncService is used to sync a service to the server
 func (l *localState) syncService(id string) error {
 	req := structs.RegisterRequest{
-		Datacenter: l.config.Datacenter,
-		Node:       l.config.NodeName,
-		Address:    l.config.AdvertiseAddr,
-		Service:    l.services[id],
+		Datacenter:   l.config.Datacenter,
+		Node:         l.config.NodeName,
+		Address:      l.config.AdvertiseAddr,
+		Service:      l.services[id],
+		WriteRequest: structs.WriteRequest{Token: l.config.ACLToken},
 	}
 	var out struct{}
 	err := l.iface.RPC("Catalog.Register", &req, &out)
 	if err == nil {
 		l.serviceStatus[id] = syncStatus{inSync: true}
 		l.logger.Printf("[INFO] agent: Synced service '%s'", id)
+	} else if strings.Contains(err.Error(), permissionDenied) {
+		l.serviceStatus[id] = syncStatus{inSync: true}
+		l.logger.Printf("[WARN] agent: Service '%s' registration blocked by ACLs", id)
+		return nil
 	}
 	return err
 }
@@ -460,17 +473,22 @@ func (l *localState) syncCheck(id string) error {
 		}
 	}
 	req := structs.RegisterRequest{
-		Datacenter: l.config.Datacenter,
-		Node:       l.config.NodeName,
-		Address:    l.config.AdvertiseAddr,
-		Service:    service,
-		Check:      l.checks[id],
+		Datacenter:   l.config.Datacenter,
+		Node:         l.config.NodeName,
+		Address:      l.config.AdvertiseAddr,
+		Service:      service,
+		Check:        l.checks[id],
+		WriteRequest: structs.WriteRequest{Token: l.config.ACLToken},
 	}
 	var out struct{}
 	err := l.iface.RPC("Catalog.Register", &req, &out)
 	if err == nil {
 		l.checkStatus[id] = syncStatus{inSync: true}
 		l.logger.Printf("[INFO] agent: Synced check '%s'", id)
+	} else if strings.Contains(err.Error(), permissionDenied) {
+		l.checkStatus[id] = syncStatus{inSync: true}
+		l.logger.Printf("[WARN] agent: Check '%s' registration blocked by ACLs", id)
+		return nil
 	}
 	return err
 }

command/agent/local_test.go

@@ -132,6 +132,103 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 	}
 }
 
+func TestAgentAntiEntropy_Services_ACLDeny(t *testing.T) {
+	conf := nextConfig()
+	conf.ACLDatacenter = "dc1"
+	conf.ACLMasterToken = "root"
+	conf.ACLDefaultPolicy = "deny"
+	dir, agent := makeAgent(t, conf)
+	defer os.RemoveAll(dir)
+	defer agent.Shutdown()
+
+	testutil.WaitForLeader(t, agent.RPC, "dc1")
+
+	// Create the ACL
+	arg := structs.ACLRequest{
+		Datacenter: "dc1",
+		Op:         structs.ACLSet,
+		ACL: structs.ACL{
+			Name:  "User token",
+			Type:  structs.ACLTypeClient,
+			Rules: testRegisterRules,
+		},
+		WriteRequest: structs.WriteRequest{Token: "root"},
+	}
+	var out string
+	if err := agent.RPC("ACL.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Update the agent ACL token, resume sync
+	conf.ACLToken = out
+
+	// Create service (Allowed)
+	srv1 := &structs.NodeService{
+		ID:      "mysql",
+		Service: "mysql",
+		Tags:    []string{"master"},
+		Port:    5000,
+	}
+	agent.state.AddService(srv1)
+
+	// Create service (Disallowed)
+	srv2 := &structs.NodeService{
+		ID:      "api",
+		Service: "api",
+		Tags:    []string{"foo"},
+		Port:    5001,
+	}
+	agent.state.AddService(srv2)
+
+	// Trigger anti-entropy run and wait
+	agent.StartSync()
+	time.Sleep(200 * time.Millisecond)
+
+	// Verify that we are in sync
+	req := structs.NodeSpecificRequest{
+		Datacenter: "dc1",
+		Node:       agent.config.NodeName,
+	}
+	var services structs.IndexedNodeServices
+	if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// We should have 2 services (consul included)
+	if len(services.NodeServices.Services) != 2 {
+		t.Fatalf("bad: %v", services.NodeServices.Services)
+	}
+
+	// All the services should match
+	for id, serv := range services.NodeServices.Services {
+		switch id {
+		case "mysql":
+			t.Fatalf("should not be permitted")
+		case "api":
+			if !reflect.DeepEqual(serv, srv2) {
+				t.Fatalf("bad: %#v %#v", serv, srv2)
+			}
+		case "consul":
+			// ignore
+		default:
+			t.Fatalf("unexpected service: %v", id)
+		}
+	}
+
+	// Check the local state
+	if len(agent.state.services) != 3 {
+		t.Fatalf("bad: %v", agent.state.services)
+	}
+	if len(agent.state.serviceStatus) != 3 {
+		t.Fatalf("bad: %v", agent.state.serviceStatus)
+	}
+	for name, status := range agent.state.serviceStatus {
+		if !status.inSync {
+			t.Fatalf("should be in sync: %v %v", name, status)
+		}
+	}
+}
+
 func TestAgentAntiEntropy_Checks(t *testing.T) {
 	conf := nextConfig()
 	dir, agent := makeAgent(t, conf)
@@ -292,7 +389,7 @@ func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) {
 		t.Fatalf("checks: %v", check)
 	}
 
-	// Update the check output! Should be defered
+	// Update the check output! Should be deferred
 	agent.state.UpdateCheck("web", structs.HealthPassing, "output")
 
 	// Should not update for 100 milliseconds
@@ -311,7 +408,7 @@ func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) {
 		}
 	}
 
-	// Wait for a defered update
+	// Wait for a deferred update
 	time.Sleep(100 * time.Millisecond)
 	if err := agent.RPC("Health.NodeChecks", &req, &checks); err != nil {
 		t.Fatalf("err: %v", err)
@@ -327,3 +424,9 @@ func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) {
 		}
 	}
 }
+
+var testRegisterRules = `
+service "api" {
+	policy = "write"
+}
+`

command/agent/log_writer.go

@@ -29,7 +29,7 @@ func NewLogWriter(buf int) *logWriter {
 	}
 }
 
-// RegisterHandler adds a log handler to recieve logs, and sends
+// RegisterHandler adds a log handler to receive logs, and sends
 // the last buffered logs to the handler
 func (l *logWriter) RegisterHandler(lh LogHandler) {
 	l.Lock()

command/agent/rpc.go

@@ -24,15 +24,17 @@ package agent
 import (
 	"bufio"
 	"fmt"
-	"github.com/hashicorp/logutils"
-	"github.com/hashicorp/serf/serf"
-	"github.com/ugorji/go/codec"
 	"io"
 	"log"
 	"net"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/go-msgpack/codec"
+	"github.com/hashicorp/logutils"
+	"github.com/hashicorp/serf/serf"
 )
 
 const (
@@ -51,6 +53,10 @@ const (
 	leaveCommand      = "leave"
 	statsCommand      = "stats"
 	reloadCommand     = "reload"
+	installKeyCommand = "install-key"
+	useKeyCommand     = "use-key"
+	removeKeyCommand  = "remove-key"
+	listKeysCommand   = "list-keys"
 )
 
 const (
@@ -103,6 +109,37 @@ type joinResponse struct {
 	Num int32
 }
 
+type keyringRequest struct {
+	Key string
+}
+
+type KeyringEntry struct {
+	Datacenter string
+	Pool       string
+	Key        string
+	Count      int
+}
+
+type KeyringMessage struct {
+	Datacenter string
+	Pool       string
+	Node       string
+	Message    string
+}
+
+type KeyringInfo struct {
+	Datacenter string
+	Pool       string
+	NumNodes   int
+	Error      string
+}
+
+type keyringResponse struct {
+	Keys     []KeyringEntry
+	Messages []KeyringMessage
+	Info     []KeyringInfo
+}
+
 type membersResponse struct {
 	Members []Member
 }
@@ -373,6 +410,9 @@ func (i *AgentRPC) handleRequest(client *rpcClient, reqHeader *requestHeader) er
 	case reloadCommand:
 		return i.handleReload(client, seq)
 
+	case installKeyCommand, useKeyCommand, removeKeyCommand, listKeysCommand:
+		return i.handleKeyring(client, seq, command)
+
 	default:
 		respHeader := responseHeader{Seq: seq, Error: unsupportedCommand}
 		client.Send(&respHeader, nil)
@@ -583,6 +623,80 @@ func (i *AgentRPC) handleReload(client *rpcClient, seq uint64) error {
 	return client.Send(&resp, nil)
 }
 
+func (i *AgentRPC) handleKeyring(client *rpcClient, seq uint64, cmd string) error {
+	var req keyringRequest
+	var queryResp *structs.KeyringResponses
+	var r keyringResponse
+	var err error
+
+	if cmd != listKeysCommand {
+		if err = client.dec.Decode(&req); err != nil {
+			return fmt.Errorf("decode failed: %v", err)
+		}
+	}
+
+	switch cmd {
+	case listKeysCommand:
+		queryResp, err = i.agent.ListKeys()
+	case installKeyCommand:
+		queryResp, err = i.agent.InstallKey(req.Key)
+	case useKeyCommand:
+		queryResp, err = i.agent.UseKey(req.Key)
+	case removeKeyCommand:
+		queryResp, err = i.agent.RemoveKey(req.Key)
+	default:
+		respHeader := responseHeader{Seq: seq, Error: unsupportedCommand}
+		client.Send(&respHeader, nil)
+		return fmt.Errorf("command '%s' not recognized", cmd)
+	}
+
+	header := responseHeader{
+		Seq:   seq,
+		Error: errToString(err),
+	}
+
+	if queryResp == nil {
+		goto SEND
+	}
+
+	for _, kr := range queryResp.Responses {
+		var pool string
+		if kr.WAN {
+			pool = "WAN"
+		} else {
+			pool = "LAN"
+		}
+		for node, message := range kr.Messages {
+			msg := KeyringMessage{
+				Datacenter: kr.Datacenter,
+				Pool:       pool,
+				Node:       node,
+				Message:    message,
+			}
+			r.Messages = append(r.Messages, msg)
+		}
+		for key, qty := range kr.Keys {
+			k := KeyringEntry{
+				Datacenter: kr.Datacenter,
+				Pool:       pool,
+				Key:        key,
+				Count:      qty,
+			}
+			r.Keys = append(r.Keys, k)
+		}
+		info := KeyringInfo{
+			Datacenter: kr.Datacenter,
+			Pool:       pool,
+			NumNodes:   kr.NumNodes,
+			Error:      kr.Error,
+		}
+		r.Info = append(r.Info, info)
+	}
+
+SEND:
+	return client.Send(&header, r)
+}
+
 // Used to convert an error to a string representation
 func errToString(err error) string {
 	if err == nil {

command/agent/rpc_client.go

@@ -3,8 +3,8 @@ package agent
 import (
 	"bufio"
 	"fmt"
+	"github.com/hashicorp/go-msgpack/codec"
 	"github.com/hashicorp/logutils"
-	"github.com/ugorji/go/codec"
 	"log"
 	"net"
 	"sync"
@@ -176,6 +176,49 @@ func (c *RPCClient) WANMembers() ([]Member, error) {
 	return resp.Members, err
 }
 
+func (c *RPCClient) ListKeys() (keyringResponse, error) {
+	header := requestHeader{
+		Command: listKeysCommand,
+		Seq:     c.getSeq(),
+	}
+	var resp keyringResponse
+	err := c.genericRPC(&header, nil, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) InstallKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: installKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) UseKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: useKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) RemoveKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: removeKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
 // Leave is used to trigger a graceful leave and shutdown
 func (c *RPCClient) Leave() error {
 	header := requestHeader{

command/agent/rpc_client_test.go

@@ -30,6 +30,10 @@ func (r *rpcParts) Close() {
 // testRPCClient returns an RPCClient connected to an RPC server that
 // serves only this connection.
 func testRPCClient(t *testing.T) *rpcParts {
+	return testRPCClientWithConfig(t, func(c *Config) {})
+}
+
+func testRPCClientWithConfig(t *testing.T, cb func(c *Config)) *rpcParts {
 	l, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -39,6 +43,8 @@ func testRPCClient(t *testing.T) *rpcParts {
 	mult := io.MultiWriter(os.Stderr, lw)
 
 	conf := nextConfig()
+	cb(conf)
+
 	dir, agent := makeAgentLog(t, conf, mult)
 	rpc := NewAgentRPC(agent, l, mult, lw)
 
@@ -232,7 +238,7 @@ func TestRPCClientMonitor(t *testing.T) {
 
 	found := false
 OUTER1:
-	for {
+	for i := 0; ; i++ {
 		select {
 		case e := <-eventCh:
 			if strings.Contains(e, "Accepted client") {
@@ -240,6 +246,10 @@ OUTER1:
 				break OUTER1
 			}
 		default:
+			if i > 100 {
+				break OUTER1
+			}
+			time.Sleep(10 * time.Millisecond)
 		}
 	}
 	if !found {
@@ -249,21 +259,179 @@ OUTER1:
 	// Join a bad thing to generate more events
 	p1.agent.JoinLAN(nil)
 
-	time.Sleep(1 * time.Second)
-
 	found = false
 OUTER2:
-	for {
+	for i := 0; ; i++ {
 		select {
 		case e := <-eventCh:
 			if strings.Contains(e, "joining") {
 				found = true
+				break OUTER2
 			}
 		default:
-			break OUTER2
+			if i > 100 {
+				break OUTER2
+			}
+			time.Sleep(10 * time.Millisecond)
 		}
 	}
 	if !found {
 		t.Fatalf("should log joining")
 	}
 }
+
+func TestRPCClientListKeys(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+		c.Datacenter = "dc1"
+	})
+	defer p1.Close()
+
+	// Key is initially installed to both wan/lan
+	keys := listKeys(t, p1.client)
+	if _, ok := keys["dc1"][key1]; !ok {
+		t.Fatalf("bad: %#v", keys)
+	}
+	if _, ok := keys["WAN"][key1]; !ok {
+		t.Fatalf("bad: %#v", keys)
+	}
+}
+
+func TestRPCClientInstallKey(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+	})
+	defer p1.Close()
+
+	// key2 is not installed yet
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; ok || num != 0 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; ok || num != 0 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+
+	// install key2
+	r, err := p1.client.InstallKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// key2 should now be installed
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+}
+
+func TestRPCClientUseKey(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+	})
+	defer p1.Close()
+
+	// add a second key to the ring
+	r, err := p1.client.InstallKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// key2 is installed
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+
+	// can't remove key1 yet
+	r, err = p1.client.RemoveKey(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringError(t, r)
+
+	// change primary key
+	r, err = p1.client.UseKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// can remove key1 now
+	r, err = p1.client.RemoveKey(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+}
+
+func TestRPCClientKeyOperation_encryptionDisabled(t *testing.T) {
+	p1 := testRPCClient(t)
+	defer p1.Close()
+
+	r, err := p1.client.ListKeys()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringError(t, r)
+}
+
+func listKeys(t *testing.T, c *RPCClient) map[string]map[string]int {
+	resp, err := c.ListKeys()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	out := make(map[string]map[string]int)
+	for _, k := range resp.Keys {
+		respID := k.Datacenter
+		if k.Pool == "WAN" {
+			respID = k.Pool
+		}
+		out[respID] = map[string]int{k.Key: k.Count}
+	}
+	return out
+}
+
+func keyringError(t *testing.T, r keyringResponse) {
+	for _, i := range r.Info {
+		if i.Error == "" {
+			t.Fatalf("no error reported from %s (%s)", i.Datacenter, i.Pool)
+		}
+	}
+}
+
+func keyringSuccess(t *testing.T, r keyringResponse) {
+	for _, i := range r.Info {
+		if i.Error != "" {
+			t.Fatalf("error from %s (%s): %s", i.Datacenter, i.Pool, i.Error)
+		}
+	}
+}

command/agent/session_endpoint.go

@@ -2,11 +2,12 @@ package agent
 
 import (
 	"fmt"
-	"github.com/hashicorp/consul/consul"
-	"github.com/hashicorp/consul/consul/structs"
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/hashicorp/consul/consul"
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 const (
@@ -32,13 +33,15 @@ func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request)
 		return nil, nil
 	}
 
-	// Default the session to our node + serf check
+	// Default the session to our node + serf check + release session invalidate behavior
 	args := structs.SessionRequest{
 		Op: structs.SessionCreate,
 		Session: structs.Session{
 			Node:      s.agent.config.NodeName,
 			Checks:    []string{consul.SerfCheckID},
 			LockDelay: 15 * time.Second,
+			Behavior:  structs.SessionKeysRelease,
+			TTL:       "",
 		},
 	}
 	s.parseDC(req, &args.Datacenter)
@@ -50,6 +53,21 @@ func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request)
 			resp.Write([]byte(fmt.Sprintf("Request decode failed: %v", err)))
 			return nil, nil
 		}
+
+		if args.Session.TTL != "" {
+			ttl, err := time.ParseDuration(args.Session.TTL)
+			if err != nil {
+				resp.WriteHeader(400)
+				resp.Write([]byte(fmt.Sprintf("Request TTL decode failed: %v", err)))
+				return nil, nil
+			}
+
+			if ttl != 0 && (ttl < structs.SessionTTLMin || ttl > structs.SessionTTLMax) {
+				resp.WriteHeader(400)
+				resp.Write([]byte(fmt.Sprintf("Request TTL '%s', must be between [%v-%v]", args.Session.TTL, structs.SessionTTLMin, structs.SessionTTLMax)))
+				return nil, nil
+			}
+		}
 	}
 
 	// Create the session, get the ID
@@ -129,6 +147,39 @@ func (s *HTTPServer) SessionDestroy(resp http.ResponseWriter, req *http.Request)
 	return true, nil
 }
 
+// SessionRenew is used to renew the TTL on an existing TTL session
+func (s *HTTPServer) SessionRenew(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	// Mandate a PUT request
+	if req.Method != "PUT" {
+		resp.WriteHeader(405)
+		return nil, nil
+	}
+
+	args := structs.SessionSpecificRequest{}
+	if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done {
+		return nil, nil
+	}
+
+	// Pull out the session id
+	args.Session = strings.TrimPrefix(req.URL.Path, "/v1/session/renew/")
+	if args.Session == "" {
+		resp.WriteHeader(400)
+		resp.Write([]byte("Missing session"))
+		return nil, nil
+	}
+
+	var out structs.IndexedSessions
+	if err := s.agent.RPC("Session.Renew", &args, &out); err != nil {
+		return nil, err
+	} else if out.Sessions == nil {
+		resp.WriteHeader(404)
+		resp.Write([]byte(fmt.Sprintf("Session id '%s' not found", args.Session)))
+		return nil, nil
+	}
+
+	return out.Sessions, nil
+}
+
 // SessionGet is used to get info for a particular session
 func (s *HTTPServer) SessionGet(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	args := structs.SessionSpecificRequest{}

command/agent/session_endpoint_test.go

@@ -59,6 +59,55 @@ func TestSessionCreate(t *testing.T) {
 	})
 }
 
+func TestSessionCreateDelete(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		// Create a health check
+		args := &structs.RegisterRequest{
+			Datacenter: "dc1",
+			Node:       srv.agent.config.NodeName,
+			Address:    "127.0.0.1",
+			Check: &structs.HealthCheck{
+				CheckID:   "consul",
+				Node:      srv.agent.config.NodeName,
+				Name:      "consul",
+				ServiceID: "consul",
+				Status:    structs.HealthPassing,
+			},
+		}
+		var out struct{}
+		if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		// Associate session with node and 2 health checks, and make it delete on session destroy
+		body := bytes.NewBuffer(nil)
+		enc := json.NewEncoder(body)
+		raw := map[string]interface{}{
+			"Name":      "my-cool-session",
+			"Node":      srv.agent.config.NodeName,
+			"Checks":    []string{consul.SerfCheckID, "consul"},
+			"LockDelay": "20s",
+			"Behavior":  structs.SessionKeysDelete,
+		}
+		enc.Encode(raw)
+
+		req, err := http.NewRequest("PUT", "/v1/session/create", body)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		resp := httptest.NewRecorder()
+		obj, err := srv.SessionCreate(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if _, ok := obj.(sessionCreateResponse); !ok {
+			t.Fatalf("should work")
+		}
+	})
+}
+
 func TestFixupLockDelay(t *testing.T) {
 	inp := map[string]interface{}{
 		"lockdelay": float64(15),
@@ -105,6 +154,50 @@ func makeTestSession(t *testing.T, srv *HTTPServer) string {
 	return sessResp.ID
 }
 
+func makeTestSessionDelete(t *testing.T, srv *HTTPServer) string {
+	// Create Session with delete behavior
+	body := bytes.NewBuffer(nil)
+	enc := json.NewEncoder(body)
+	raw := map[string]interface{}{
+		"Behavior": "delete",
+	}
+	enc.Encode(raw)
+
+	req, err := http.NewRequest("PUT", "/v1/session/create", body)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	resp := httptest.NewRecorder()
+	obj, err := srv.SessionCreate(resp, req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	sessResp := obj.(sessionCreateResponse)
+	return sessResp.ID
+}
+
+func makeTestSessionTTL(t *testing.T, srv *HTTPServer, ttl string) string {
+	// Create Session with TTL
+	body := bytes.NewBuffer(nil)
+	enc := json.NewEncoder(body)
+	raw := map[string]interface{}{
+		"TTL": ttl,
+	}
+	enc.Encode(raw)
+
+	req, err := http.NewRequest("PUT", "/v1/session/create", body)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	resp := httptest.NewRecorder()
+	obj, err := srv.SessionCreate(resp, req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	sessResp := obj.(sessionCreateResponse)
+	return sessResp.ID
+}
+
 func TestSessionDestroy(t *testing.T) {
 	httpTest(t, func(srv *HTTPServer) {
 		id := makeTestSession(t, srv)
@@ -121,6 +214,206 @@ func TestSessionDestroy(t *testing.T) {
 	})
 }
 
+func TestSessionTTL(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		TTL := "10s" // use the minimum legal ttl
+		ttl := 10 * time.Second
+
+		id := makeTestSessionTTL(t, srv, TTL)
+
+		req, err := http.NewRequest("GET",
+			"/v1/session/info/"+id, nil)
+		resp := httptest.NewRecorder()
+		obj, err := srv.SessionGet(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok := obj.(structs.Sessions)
+		if !ok {
+			t.Fatalf("should work")
+		}
+		if len(respObj) != 1 {
+			t.Fatalf("bad: %v", respObj)
+		}
+		if respObj[0].TTL != TTL {
+			t.Fatalf("Incorrect TTL: %s", respObj[0].TTL)
+		}
+
+		time.Sleep(ttl*structs.SessionTTLMultiplier + ttl)
+
+		req, err = http.NewRequest("GET",
+			"/v1/session/info/"+id, nil)
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionGet(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok = obj.(structs.Sessions)
+		if len(respObj) != 0 {
+			t.Fatalf("session '%s' should have been destroyed", id)
+		}
+	})
+}
+
+func TestSessionBadTTL(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		badTTL := "10z"
+
+		// Create Session with illegal TTL
+		body := bytes.NewBuffer(nil)
+		enc := json.NewEncoder(body)
+		raw := map[string]interface{}{
+			"TTL": badTTL,
+		}
+		enc.Encode(raw)
+
+		req, err := http.NewRequest("PUT", "/v1/session/create", body)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		resp := httptest.NewRecorder()
+		obj, err := srv.SessionCreate(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if obj != nil {
+			t.Fatalf("illegal TTL '%s' allowed", badTTL)
+		}
+		if resp.Code != 400 {
+			t.Fatalf("Bad response code, should be 400")
+		}
+
+		// less than SessionTTLMin
+		body = bytes.NewBuffer(nil)
+		enc = json.NewEncoder(body)
+		raw = map[string]interface{}{
+			"TTL": "5s",
+		}
+		enc.Encode(raw)
+
+		req, err = http.NewRequest("PUT", "/v1/session/create", body)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionCreate(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if obj != nil {
+			t.Fatalf("illegal TTL '%s' allowed", badTTL)
+		}
+		if resp.Code != 400 {
+			t.Fatalf("Bad response code, should be 400")
+		}
+
+		// more than SessionTTLMax
+		body = bytes.NewBuffer(nil)
+		enc = json.NewEncoder(body)
+		raw = map[string]interface{}{
+			"TTL": "4000s",
+		}
+		enc.Encode(raw)
+
+		req, err = http.NewRequest("PUT", "/v1/session/create", body)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionCreate(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if obj != nil {
+			t.Fatalf("illegal TTL '%s' allowed", badTTL)
+		}
+		if resp.Code != 400 {
+			t.Fatalf("Bad response code, should be 400")
+		}
+	})
+}
+
+func TestSessionTTLRenew(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		TTL := "10s" // use the minimum legal ttl
+		ttl := 10 * time.Second
+
+		id := makeTestSessionTTL(t, srv, TTL)
+
+		req, err := http.NewRequest("GET",
+			"/v1/session/info/"+id, nil)
+		resp := httptest.NewRecorder()
+		obj, err := srv.SessionGet(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok := obj.(structs.Sessions)
+		if !ok {
+			t.Fatalf("should work")
+		}
+		if len(respObj) != 1 {
+			t.Fatalf("bad: %v", respObj)
+		}
+		if respObj[0].TTL != TTL {
+			t.Fatalf("Incorrect TTL: %s", respObj[0].TTL)
+		}
+
+		// Sleep to consume some time before renew
+		time.Sleep(ttl * (structs.SessionTTLMultiplier / 2))
+
+		req, err = http.NewRequest("PUT",
+			"/v1/session/renew/"+id, nil)
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionRenew(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok = obj.(structs.Sessions)
+		if !ok {
+			t.Fatalf("should work")
+		}
+		if len(respObj) != 1 {
+			t.Fatalf("bad: %v", respObj)
+		}
+
+		// Sleep for ttl * TTL Multiplier
+		time.Sleep(ttl * structs.SessionTTLMultiplier)
+
+		req, err = http.NewRequest("GET",
+			"/v1/session/info/"+id, nil)
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionGet(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok = obj.(structs.Sessions)
+		if !ok {
+			t.Fatalf("session '%s' should have renewed", id)
+		}
+		if len(respObj) != 1 {
+			t.Fatalf("session '%s' should have renewed", id)
+		}
+
+		// now wait for timeout and expect session to get destroyed
+		time.Sleep(ttl * structs.SessionTTLMultiplier)
+
+		req, err = http.NewRequest("GET",
+			"/v1/session/info/"+id, nil)
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionGet(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		respObj, ok = obj.(structs.Sessions)
+		if !ok {
+			t.Fatalf("session '%s' should have destroyed", id)
+		}
+		if len(respObj) != 0 {
+			t.Fatalf("session '%s' should have destroyed", id)
+		}
+	})
+}
+
 func TestSessionGet(t *testing.T) {
 	httpTest(t, func(srv *HTTPServer) {
 		id := makeTestSession(t, srv)
@@ -188,3 +481,45 @@ func TestSessionsForNode(t *testing.T) {
 		}
 	})
 }
+
+func TestSessionDeleteDestroy(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		id := makeTestSessionDelete(t, srv)
+
+		// now create a new key for the session and acquire it
+		buf := bytes.NewBuffer([]byte("test"))
+		req, err := http.NewRequest("PUT", "/v1/kv/ephemeral?acquire="+id, buf)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		resp := httptest.NewRecorder()
+		obj, err := srv.KVSEndpoint(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if res := obj.(bool); !res {
+			t.Fatalf("should work")
+		}
+
+		// now destroy the session, this should delete the key created above
+		req, err = http.NewRequest("PUT", "/v1/session/destroy/"+id, nil)
+		resp = httptest.NewRecorder()
+		obj, err = srv.SessionDestroy(resp, req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		if resp := obj.(bool); !resp {
+			t.Fatalf("should work")
+		}
+
+		// Verify that the key is gone
+		req, _ = http.NewRequest("GET", "/v1/kv/ephemeral", nil)
+		resp = httptest.NewRecorder()
+		obj, _ = srv.KVSEndpoint(resp, req)
+		res, found := obj.(structs.DirEntries)
+		if found || len(res) != 0 {
+			t.Fatalf("bad: %v found, should be nothing", res)
+		}
+	})
+}

command/agent/syslog_test.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/hashicorp/go-syslog"
@@ -8,6 +9,9 @@ import (
 )
 
 func TestSyslogFilter(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.SkipNow()
+	}
 	l, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, "LOCAL0", "consul")
 	if err != nil {
 		t.Fatalf("err: %s", err)

command/agent/ui_endpoint_test.go

@@ -41,7 +41,7 @@ func TestUiIndex(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Verify teh response
+	// Verify the response
 	if resp.StatusCode != 200 {
 		t.Fatalf("bad: %v", resp)
 	}

command/agent/util.go

@@ -11,7 +11,7 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/ugorji/go/codec"
+	"github.com/hashicorp/go-msgpack/codec"
 )
 
 const (

command/exec.go

@@ -384,7 +384,6 @@ func (c *ExecCommand) streamResults(doneCh chan struct{}, ackCh chan rExecAck, h
 			}
 		}
 	}
-	return
 
 ERR_EXIT:
 	select {

command/keyring.go

@@ -0,0 +1,216 @@
+package command
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/consul/command/agent"
+	"github.com/mitchellh/cli"
+)
+
+// KeyringCommand is a Command implementation that handles querying, installing,
+// and removing gossip encryption keys from a keyring.
+type KeyringCommand struct {
+	Ui cli.Ui
+}
+
+func (c *KeyringCommand) Run(args []string) int {
+	var installKey, useKey, removeKey string
+	var listKeys bool
+
+	cmdFlags := flag.NewFlagSet("keys", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
+
+	cmdFlags.StringVar(&installKey, "install", "", "install key")
+	cmdFlags.StringVar(&useKey, "use", "", "use key")
+	cmdFlags.StringVar(&removeKey, "remove", "", "remove key")
+	cmdFlags.BoolVar(&listKeys, "list", false, "list keys")
+
+	rpcAddr := RPCAddrFlag(cmdFlags)
+	if err := cmdFlags.Parse(args); err != nil {
+		return 1
+	}
+
+	c.Ui = &cli.PrefixedUi{
+		OutputPrefix: "",
+		InfoPrefix:   "==> ",
+		ErrorPrefix:  "",
+		Ui:           c.Ui,
+	}
+
+	// Only accept a single argument
+	found := listKeys
+	for _, arg := range []string{installKey, useKey, removeKey} {
+		if found && len(arg) > 0 {
+			c.Ui.Error("Only a single action is allowed")
+			return 1
+		}
+		found = found || len(arg) > 0
+	}
+
+	// Fail fast if no actionable args were passed
+	if !found {
+		c.Ui.Error(c.Help())
+		return 1
+	}
+
+	// All other operations will require a client connection
+	client, err := RPCClient(*rpcAddr)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
+		return 1
+	}
+	defer client.Close()
+
+	if listKeys {
+		c.Ui.Info("Gathering installed encryption keys...")
+		r, err := client.ListKeys()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		if rval := c.handleResponse(r.Info, r.Messages); rval != 0 {
+			return rval
+		}
+		c.handleList(r.Info, r.Keys)
+		return 0
+	}
+
+	if installKey != "" {
+		c.Ui.Info("Installing new gossip encryption key...")
+		r, err := client.InstallKey(installKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	if useKey != "" {
+		c.Ui.Info("Changing primary gossip encryption key...")
+		r, err := client.UseKey(useKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	if removeKey != "" {
+		c.Ui.Info("Removing gossip encryption key...")
+		r, err := client.RemoveKey(removeKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	// Should never make it here
+	return 0
+}
+
+func (c *KeyringCommand) handleResponse(
+	info []agent.KeyringInfo,
+	messages []agent.KeyringMessage) int {
+
+	var rval int
+
+	for _, i := range info {
+		if i.Error != "" {
+			pool := i.Pool
+			if pool != "WAN" {
+				pool = i.Datacenter + " (LAN)"
+			}
+
+			c.Ui.Error("")
+			c.Ui.Error(fmt.Sprintf("%s error: %s", pool, i.Error))
+
+			for _, msg := range messages {
+				if msg.Datacenter != i.Datacenter || msg.Pool != i.Pool {
+					continue
+				}
+				c.Ui.Error(fmt.Sprintf("  %s: %s", msg.Node, msg.Message))
+			}
+			rval = 1
+		}
+	}
+
+	if rval == 0 {
+		c.Ui.Info("Done!")
+	}
+
+	return rval
+}
+
+func (c *KeyringCommand) handleList(
+	info []agent.KeyringInfo,
+	keys []agent.KeyringEntry) {
+
+	installed := make(map[string]map[string][]int)
+	for _, key := range keys {
+		var nodes int
+		for _, i := range info {
+			if i.Datacenter == key.Datacenter && i.Pool == key.Pool {
+				nodes = i.NumNodes
+			}
+		}
+
+		pool := key.Pool
+		if pool != "WAN" {
+			pool = key.Datacenter + " (LAN)"
+		}
+
+		if _, ok := installed[pool]; !ok {
+			installed[pool] = map[string][]int{key.Key: []int{key.Count, nodes}}
+		} else {
+			installed[pool][key.Key] = []int{key.Count, nodes}
+		}
+	}
+
+	for pool, keys := range installed {
+		c.Ui.Output("")
+		c.Ui.Output(pool + ":")
+		for key, num := range keys {
+			c.Ui.Output(fmt.Sprintf("  %s [%d/%d]", key, num[0], num[1]))
+		}
+	}
+}
+
+func (c *KeyringCommand) Help() string {
+	helpText := `
+Usage: consul keyring [options]
+
+  Manages encryption keys used for gossip messages. Gossip encryption is
+  optional. When enabled, this command may be used to examine active encryption
+  keys in the cluster, add new keys, and remove old ones. When combined, this
+  functionality provides the ability to perform key rotation cluster-wide,
+  without disrupting the cluster.
+
+  All operations performed by this command can only be run against server nodes,
+  and affect both the LAN and WAN keyrings in lock-step.
+
+  All variations of the keyring command return 0 if all nodes reply and there
+  are no errors. If any node fails to reply or reports failure, the exit code
+  will be 1.
+
+Options:
+
+  -install=<key>            Install a new encryption key. This will broadcast
+                            the new key to all members in the cluster.
+  -use=<key>                Change the primary encryption key, which is used to
+                            encrypt messages. The key must already be installed
+                            before this operation can succeed.
+  -remove=<key>             Remove the given key from the cluster. This
+                            operation may only be performed on keys which are
+                            not currently the primary key.
+  -list                     List all keys currently in use within the cluster.
+  -rpc-addr=127.0.0.1:8400  RPC address of the Consul agent.
+`
+	return strings.TrimSpace(helpText)
+}
+
+func (c *KeyringCommand) Synopsis() string {
+	return "Manages gossip layer encryption keys"
+}

command/keyring_test.go

@@ -0,0 +1,136 @@
+package command
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/consul/command/agent"
+	"github.com/mitchellh/cli"
+)
+
+func TestKeyringCommand_implements(t *testing.T) {
+	var _ cli.Command = &KeyringCommand{}
+}
+
+func TestKeyringCommandRun(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "kZyFABeAmc64UMTrm9XuKA=="
+
+	// Begin with a single key
+	a1 := testAgentWithConfig(t, func(c *agent.Config) {
+		c.EncryptKey = key1
+	})
+	defer a1.Shutdown()
+
+	// The LAN and WAN keyrings were initialized with key1
+	out := listKeys(t, a1.addr)
+	if !strings.Contains(out, "dc1 (LAN):\n  "+key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if !strings.Contains(out, "WAN:\n  "+key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if strings.Contains(out, key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+
+	// Install the second key onto the keyring
+	installKey(t, a1.addr, key2)
+
+	// Both keys should be present
+	out = listKeys(t, a1.addr)
+	for _, key := range []string{key1, key2} {
+		if !strings.Contains(out, key) {
+			t.Fatalf("bad: %#v", out)
+		}
+	}
+
+	// Rotate to key2, remove key1
+	useKey(t, a1.addr, key2)
+	removeKey(t, a1.addr, key1)
+
+	// Only key2 is present now
+	out = listKeys(t, a1.addr)
+	if !strings.Contains(out, "dc1 (LAN):\n  "+key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if !strings.Contains(out, "WAN:\n  "+key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if strings.Contains(out, key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+}
+
+func TestKeyringCommandRun_help(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+	code := c.Run(nil)
+	if code != 1 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	// Test that we didn't actually try to dial the RPC server.
+	if !strings.Contains(ui.ErrorWriter.String(), "Usage:") {
+		t.Fatalf("bad: %#v", ui.ErrorWriter.String())
+	}
+}
+
+func TestKeyringCommandRun_failedConnection(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+	args := []string{"-list", "-rpc-addr=127.0.0.1:0"}
+	code := c.Run(args)
+	if code != 1 {
+		t.Fatalf("bad: %d, %#v", code, ui.ErrorWriter.String())
+	}
+	if !strings.Contains(ui.ErrorWriter.String(), "dial") {
+		t.Fatalf("bad: %#v", ui.OutputWriter.String())
+	}
+}
+
+func listKeys(t *testing.T, addr string) string {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-list", "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	return ui.OutputWriter.String()
+}
+
+func installKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-install=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}
+
+func useKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-use=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}
+
+func removeKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-remove=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}

command/util_test.go

@@ -39,6 +39,10 @@ func (a *agentWrapper) Shutdown() {
 }
 
 func testAgent(t *testing.T) *agentWrapper {
+	return testAgentWithConfig(t, func(c *agent.Config) {})
+}
+
+func testAgentWithConfig(t *testing.T, cb func(c *agent.Config)) *agentWrapper {
 	l, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -48,6 +52,7 @@ func testAgent(t *testing.T) *agentWrapper {
 	mult := io.MultiWriter(os.Stderr, lw)
 
 	conf := nextConfig()
+	cb(conf)
 
 	dir, err := ioutil.TempDir("", "agent")
 	if err != nil {
@@ -63,19 +68,25 @@ func testAgent(t *testing.T) *agentWrapper {
 
 	rpc := agent.NewAgentRPC(a, l, mult, lw)
 
+	conf.Addresses.HTTP = "127.0.0.1"
 	httpAddr := fmt.Sprintf("127.0.0.1:%d", conf.Ports.HTTP)
-	http, err := agent.NewHTTPServer(a, "", false, os.Stderr, httpAddr)
+	http, err := agent.NewHTTPServers(a, conf, os.Stderr)
 	if err != nil {
 		os.RemoveAll(dir)
 		t.Fatalf(fmt.Sprintf("err: %v", err))
 	}
 
+	if http == nil || len(http) == 0 {
+		os.RemoveAll(dir)
+		t.Fatalf(fmt.Sprintf("Could not create HTTP server to listen on: %s", httpAddr))
+	}
+
 	return &agentWrapper{
 		dir:      dir,
 		config:   conf,
 		agent:    a,
 		rpc:      rpc,
-		http:     http,
+		http:     http[0],
 		addr:     l.Addr().String(),
 		httpAddr: httpAddr,
 	}
@@ -92,6 +103,7 @@ func nextConfig() *agent.Config {
 	conf.Server = true
 
 	conf.Ports.HTTP = 10000 + 10*idx
+	conf.Ports.HTTPS = 10401 + 10*idx
 	conf.Ports.RPC = 10100 + 10*idx
 	conf.Ports.SerfLan = 10201 + 10*idx
 	conf.Ports.SerfWan = 10202 + 10*idx

command/version.go

@@ -21,7 +21,7 @@ func (c *VersionCommand) Help() string {
 
 func (c *VersionCommand) Run(_ []string) int {
 	var versionString bytes.Buffer
-	fmt.Fprintf(&versionString, "Consul v%s", c.Version)
+	fmt.Fprintf(&versionString, "Consul %s", c.Version)
 	if c.VersionPrerelease != "" {
 		fmt.Fprintf(&versionString, ".%s", c.VersionPrerelease)
 

command/watch.go

@@ -144,14 +144,18 @@ func (c *WatchCommand) Run(args []string) int {
 	}
 
 	// Setup handler
-	errExit := false
+
+	// errExit:
+	//	0: false
+	//	1: true
+	errExit := 0
 	if script == "" {
 		wp.Handler = func(idx uint64, data interface{}) {
 			defer wp.Stop()
 			buf, err := json.MarshalIndent(data, "", "    ")
 			if err != nil {
 				c.Ui.Error(fmt.Sprintf("Error encoding output: %s", err))
-				errExit = true
+				errExit = 1
 			}
 			c.Ui.Output(string(buf))
 		}
@@ -186,7 +190,7 @@ func (c *WatchCommand) Run(args []string) int {
 			return
 		ERR:
 			wp.Stop()
-			errExit = true
+			errExit = 1
 		}
 	}
 
@@ -203,12 +207,7 @@ func (c *WatchCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Handle an error exit
-	if errExit {
-		return 1
-	} else {
-		return 0
-	}
+	return errExit
 }
 
 func (c *WatchCommand) Synopsis() string {

commands.go

@@ -1,14 +1,15 @@
 package main
 
 import (
+	"os"
+	"os/signal"
+
 	"github.com/hashicorp/consul/command"
 	"github.com/hashicorp/consul/command/agent"
 	"github.com/mitchellh/cli"
-	"os"
-	"os/signal"
 )
 
-// Commands is the mapping of all the available Serf commands.
+// Commands is the mapping of all the available Consul commands.
 var Commands map[string]cli.CommandFactory
 
 func init() {
@@ -56,6 +57,12 @@ func init() {
 			}, nil
 		},
 
+		"keyring": func() (cli.Command, error) {
+			return &command.KeyringCommand{
+				Ui: ui,
+			}, nil
+		},
+
 		"leave": func() (cli.Command, error) {
 			return &command.LeaveCommand{
 				Ui: ui,
@@ -88,10 +95,19 @@ func init() {
 		},
 
 		"version": func() (cli.Command, error) {
+			ver := Version
+			rel := VersionPrerelease
+			if GitDescribe != "" {
+				ver = GitDescribe
+			}
+			if GitDescribe == "" && rel == "" {
+				rel = "dev"
+			}
+
 			return &command.VersionCommand{
 				Revision:          GitCommit,
-				Version:           Version,
-				VersionPrerelease: VersionPrerelease,
+				Version:           ver,
+				VersionPrerelease: rel,
 				Ui:                ui,
 			}, nil
 		},

consul/catalog_endpoint.go

@@ -2,10 +2,11 @@ package consul
 
 import (
 	"fmt"
-	"github.com/armon/go-metrics"
-	"github.com/hashicorp/consul/consul/structs"
 	"sort"
 	"time"
+
+	"github.com/armon/go-metrics"
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 // Catalog endpoint is used to manipulate the service catalog
@@ -35,6 +36,20 @@ func (c *Catalog) Register(args *structs.RegisterRequest, reply *struct{}) error
 		if args.Service.ID != "" && args.Service.Service == "" {
 			return fmt.Errorf("Must provide service name with ID")
 		}
+
+		// Apply the ACL policy if any
+		// The 'consul' service is excluded since it is managed
+		// automatically internally.
+		if args.Service.Service != ConsulServiceName {
+			acl, err := c.srv.resolveToken(args.Token)
+			if err != nil {
+				return err
+			} else if acl != nil && !acl.ServiceWrite(args.Service.Service) {
+				c.srv.logger.Printf("[WARN] consul.catalog: Register of service '%s' on '%s' denied due to ACLs",
+					args.Service.Service, args.Node)
+				return permissionDeniedErr
+			}
+		}
 	}
 
 	if args.Check != nil {

consul/catalog_endpoint_test.go

@@ -45,6 +45,61 @@ func TestCatalogRegister(t *testing.T) {
 	})
 }
 
+func TestCatalogRegister_ACLDeny(t *testing.T) {
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.ACLDatacenter = "dc1"
+		c.ACLMasterToken = "root"
+		c.ACLDefaultPolicy = "deny"
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+
+	// Create the ACL
+	arg := structs.ACLRequest{
+		Datacenter: "dc1",
+		Op:         structs.ACLSet,
+		ACL: structs.ACL{
+			Name:  "User token",
+			Type:  structs.ACLTypeClient,
+			Rules: testRegisterRules,
+		},
+		WriteRequest: structs.WriteRequest{Token: "root"},
+	}
+	var out string
+	if err := client.Call("ACL.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	id := out
+
+	argR := structs.RegisterRequest{
+		Datacenter: "dc1",
+		Node:       "foo",
+		Address:    "127.0.0.1",
+		Service: &structs.NodeService{
+			Service: "db",
+			Tags:    []string{"master"},
+			Port:    8000,
+		},
+		WriteRequest: structs.WriteRequest{Token: id},
+	}
+	var outR struct{}
+
+	err := client.Call("Catalog.Register", &argR, &outR)
+	if err == nil || !strings.Contains(err.Error(), permissionDenied) {
+		t.Fatalf("err: %v", err)
+	}
+
+	argR.Service.Service = "foo"
+	err = client.Call("Catalog.Register", &argR, &outR)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+}
+
 func TestCatalogRegister_ForwardLeader(t *testing.T) {
 	dir1, s1 := testServer(t)
 	defer os.RemoveAll(dir1)
@@ -722,3 +777,9 @@ func TestCatalogRegister_FailedCase1(t *testing.T) {
 		t.Fatalf("Bad: %v", out2)
 	}
 }
+
+var testRegisterRules = `
+service "foo" {
+	policy = "write"
+}
+`

consul/client.go

@@ -93,7 +93,7 @@ func NewClient(config *Config) (*Client, error) {
 	// Create the tlsConfig
 	var tlsConfig *tls.Config
 	var err error
-	if tlsConfig, err = config.OutgoingTLSConfig(); err != nil {
+	if tlsConfig, err = config.tlsConfig().OutgoingTLSConfig(); err != nil {
 		return nil, err
 	}
 
@@ -206,6 +206,16 @@ func (c *Client) UserEvent(name string, payload []byte) error {
 	return c.serf.UserEvent(userEventName(name), payload, false)
 }
 
+// KeyManagerLAN returns the LAN Serf keyring manager
+func (c *Client) KeyManagerLAN() *serf.KeyManager {
+	return c.serf.KeyManager()
+}
+
+// Encrypted determines if gossip is encrypted
+func (c *Client) Encrypted() bool {
+	return c.serf.EncryptionEnabled()
+}
+
 // lanEventHandler is used to handle events from the lan Serf cluster
 func (c *Client) lanEventHandler() {
 	for {

consul/client_test.go

@@ -269,3 +269,23 @@ func TestClientServer_UserEvent(t *testing.T) {
 		t.Fatalf("missing events")
 	}
 }
+
+func TestClient_Encrypted(t *testing.T) {
+	dir1, c1 := testClient(t)
+	defer os.RemoveAll(dir1)
+	defer c1.Shutdown()
+
+	key := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	dir2, c2 := testClientWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = key
+	})
+	defer os.RemoveAll(dir2)
+	defer c2.Shutdown()
+
+	if c1.Encrypted() {
+		t.Fatalf("should not be encrypted")
+	}
+	if !c2.Encrypted() {
+		t.Fatalf("should be encrypted")
+	}
+}

consul/config.go

@@ -1,15 +1,13 @@
 package consul
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"time"
 
+	"github.com/hashicorp/consul/tlsutil"
 	"github.com/hashicorp/memberlist"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/serf/serf"
@@ -199,169 +197,6 @@ func (c *Config) CheckACL() error {
 	return nil
 }
 
-// AppendCA opens and parses the CA file and adds the certificates to
-// the provided CertPool.
-func (c *Config) AppendCA(pool *x509.CertPool) error {
-	if c.CAFile == "" {
-		return nil
-	}
-
-	// Read the file
-	data, err := ioutil.ReadFile(c.CAFile)
-	if err != nil {
-		return fmt.Errorf("Failed to read CA file: %v", err)
-	}
-
-	if !pool.AppendCertsFromPEM(data) {
-		return fmt.Errorf("Failed to parse any CA certificates")
-	}
-
-	return nil
-}
-
-// KeyPair is used to open and parse a certificate and key file
-func (c *Config) KeyPair() (*tls.Certificate, error) {
-	if c.CertFile == "" || c.KeyFile == "" {
-		return nil, nil
-	}
-	cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to load cert/key pair: %v", err)
-	}
-	return &cert, err
-}
-
-// OutgoingTLSConfig generates a TLS configuration for outgoing
-// requests. It will return a nil config if this configuration should
-// not use TLS for outgoing connections.
-func (c *Config) OutgoingTLSConfig() (*tls.Config, error) {
-	if !c.VerifyOutgoing {
-		return nil, nil
-	}
-	// Create the tlsConfig
-	tlsConfig := &tls.Config{
-		RootCAs:            x509.NewCertPool(),
-		InsecureSkipVerify: true,
-	}
-	if c.ServerName != "" {
-		tlsConfig.ServerName = c.ServerName
-		tlsConfig.InsecureSkipVerify = false
-	}
-
-	// Ensure we have a CA if VerifyOutgoing is set
-	if c.VerifyOutgoing && c.CAFile == "" {
-		return nil, fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!")
-	}
-
-	// Parse the CA cert if any
-	err := c.AppendCA(tlsConfig.RootCAs)
-	if err != nil {
-		return nil, err
-	}
-
-	// Add cert/key
-	cert, err := c.KeyPair()
-	if err != nil {
-		return nil, err
-	} else if cert != nil {
-		tlsConfig.Certificates = []tls.Certificate{*cert}
-	}
-
-	return tlsConfig, nil
-}
-
-// Wrap a net.Conn into a client tls connection, performing any
-// additional verification as needed.
-//
-// As of go 1.3, crypto/tls only supports either doing no certificate
-// verification, or doing full verification including of the peer's
-// DNS name. For consul, we want to validate that the certificate is
-// signed by a known CA, but because consul doesn't use DNS names for
-// node names, we don't verify the certificate DNS names. Since go 1.3
-// no longer supports this mode of operation, we have to do it
-// manually.
-func wrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
-	var err error
-	var tlsConn *tls.Conn
-
-	tlsConn = tls.Client(conn, tlsConfig)
-
-	// If crypto/tls is doing verification, there's no need to do
-	// our own.
-	if tlsConfig.InsecureSkipVerify == false {
-		return tlsConn, nil
-	}
-
-	if err = tlsConn.Handshake(); err != nil {
-		tlsConn.Close()
-		return nil, err
-	}
-
-	// The following is lightly-modified from the doFullHandshake
-	// method in crypto/tls's handshake_client.go.
-	opts := x509.VerifyOptions{
-		Roots:         tlsConfig.RootCAs,
-		CurrentTime:   time.Now(),
-		DNSName:       "",
-		Intermediates: x509.NewCertPool(),
-	}
-
-	certs := tlsConn.ConnectionState().PeerCertificates
-	for i, cert := range certs {
-		if i == 0 {
-			continue
-		}
-		opts.Intermediates.AddCert(cert)
-	}
-
-	_, err = certs[0].Verify(opts)
-	if err != nil {
-		tlsConn.Close()
-		return nil, err
-	}
-
-	return tlsConn, err
-}
-
-// IncomingTLSConfig generates a TLS configuration for incoming requests
-func (c *Config) IncomingTLSConfig() (*tls.Config, error) {
-	// Create the tlsConfig
-	tlsConfig := &tls.Config{
-		ServerName: c.ServerName,
-		ClientCAs:  x509.NewCertPool(),
-		ClientAuth: tls.NoClientCert,
-	}
-	if tlsConfig.ServerName == "" {
-		tlsConfig.ServerName = c.NodeName
-	}
-
-	// Parse the CA cert if any
-	err := c.AppendCA(tlsConfig.ClientCAs)
-	if err != nil {
-		return nil, err
-	}
-
-	// Add cert/key
-	cert, err := c.KeyPair()
-	if err != nil {
-		return nil, err
-	} else if cert != nil {
-		tlsConfig.Certificates = []tls.Certificate{*cert}
-	}
-
-	// Check if we require verification
-	if c.VerifyIncoming {
-		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
-		if c.CAFile == "" {
-			return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!")
-		}
-		if cert == nil {
-			return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!")
-		}
-	}
-	return tlsConfig, nil
-}
-
 // DefaultConfig is used to return a sane default configuration
 func DefaultConfig() *Config {
 	hostname, err := os.Hostname()
@@ -400,3 +235,16 @@ func DefaultConfig() *Config {
 
 	return conf
 }
+
+func (c *Config) tlsConfig() *tlsutil.Config {
+	tlsConf := &tlsutil.Config{
+		VerifyIncoming: c.VerifyIncoming,
+		VerifyOutgoing: c.VerifyOutgoing,
+		CAFile:         c.CAFile,
+		CertFile:       c.CertFile,
+		KeyFile:        c.KeyFile,
+		NodeName:       c.NodeName,
+		ServerName:     c.ServerName}
+
+	return tlsConf
+}

consul/fsm.go

@@ -1,6 +1,7 @@
 package consul
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -8,8 +9,8 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/go-msgpack/codec"
 	"github.com/hashicorp/raft"
-	"github.com/ugorji/go/codec"
 )
 
 // consulFSM implements a finite state machine that is used
@@ -164,10 +165,10 @@ func (c *consulFSM) applyKVSOperation(buf []byte, index uint64) interface{} {
 			return act
 		}
 	default:
-		c.logger.Printf("[WARN] consul.fsm: Invalid KVS operation '%s'", req.Op)
-		return fmt.Errorf("Invalid KVS operation '%s'", req.Op)
+		err := errors.New(fmt.Sprintf("Invalid KVS operation '%s'", req.Op))
+		c.logger.Printf("[WARN] consul.fsm: %v", err)
+		return err
 	}
-	return nil
 }
 
 func (c *consulFSM) applySessionOperation(buf []byte, index uint64) interface{} {
@@ -188,7 +189,6 @@ func (c *consulFSM) applySessionOperation(buf []byte, index uint64) interface{}
 		c.logger.Printf("[WARN] consul.fsm: Invalid Session operation '%s'", req.Op)
 		return fmt.Errorf("Invalid Session operation '%s'", req.Op)
 	}
-	return nil
 }
 
 func (c *consulFSM) applyACLOperation(buf []byte, index uint64) interface{} {
@@ -211,7 +211,6 @@ func (c *consulFSM) applyACLOperation(buf []byte, index uint64) interface{} {
 		c.logger.Printf("[WARN] consul.fsm: Invalid ACL operation '%s'", req.Op)
 		return fmt.Errorf("Invalid ACL operation '%s'", req.Op)
 	}
-	return nil
 }
 
 func (c *consulFSM) Snapshot() (raft.FSMSnapshot, error) {
@@ -443,7 +442,6 @@ func (s *consulSnapshot) persistKV(sink raft.SnapshotSink,
 			return err
 		}
 	}
-	return nil
 }
 
 func (s *consulSnapshot) Release() {

consul/fsm_test.go

@@ -414,23 +414,38 @@ func TestFSM_SnapshotRestore(t *testing.T) {
 		t.Fatalf("bad: %v", d)
 	}
 
+	// Verify the index is restored
+	idx, _, err := fsm.state.KVSListKeys("/blah", "")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if idx <= 1 {
+		t.Fatalf("bad index: %d", idx)
+	}
+
 	// Verify session is restored
-	_, s, err := fsm.state.SessionGet(session.ID)
+	idx, s, err := fsm.state.SessionGet(session.ID)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 	if s.Node != "foo" {
 		t.Fatalf("bad: %v", s)
 	}
+	if idx <= 1 {
+		t.Fatalf("bad index: %d", idx)
+	}
 
 	// Verify ACL is restored
-	_, a, err := fsm.state.ACLGet(acl.ID)
+	idx, a, err := fsm.state.ACLGet(acl.ID)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 	if a.Name != "User Token" {
 		t.Fatalf("bad: %v", a)
 	}
+	if idx <= 1 {
+		t.Fatalf("bad index: %d", idx)
+	}
 }
 
 func TestFSM_KVSSet(t *testing.T) {

consul/internal_endpoint.go

@@ -2,6 +2,7 @@ package consul
 
 import (
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/serf/serf"
 )
 
 // Internal endpoint is used to query the miscellaneous info that
@@ -62,3 +63,64 @@ func (m *Internal) EventFire(args *structs.EventFireRequest,
 	// Fire the event
 	return m.srv.UserEvent(args.Name, args.Payload)
 }
+
+// KeyringOperation will query the WAN and LAN gossip keyrings of all nodes.
+func (m *Internal) KeyringOperation(
+	args *structs.KeyringRequest,
+	reply *structs.KeyringResponses) error {
+
+	// Only perform WAN keyring querying and RPC forwarding once
+	if !args.Forwarded {
+		args.Forwarded = true
+		m.executeKeyringOp(args, reply, true)
+		return m.srv.globalRPC("Internal.KeyringOperation", args, reply)
+	}
+
+	// Query the LAN keyring of this node's DC
+	m.executeKeyringOp(args, reply, false)
+	return nil
+}
+
+// executeKeyringOp executes the appropriate keyring-related function based on
+// the type of keyring operation in the request. It takes the KeyManager as an
+// argument, so it can handle any operation for either LAN or WAN pools.
+func (m *Internal) executeKeyringOp(
+	args *structs.KeyringRequest,
+	reply *structs.KeyringResponses,
+	wan bool) {
+
+	var serfResp *serf.KeyResponse
+	var err error
+	var mgr *serf.KeyManager
+
+	if wan {
+		mgr = m.srv.KeyManagerWAN()
+	} else {
+		mgr = m.srv.KeyManagerLAN()
+	}
+
+	switch args.Operation {
+	case structs.KeyringList:
+		serfResp, err = mgr.ListKeys()
+	case structs.KeyringInstall:
+		serfResp, err = mgr.InstallKey(args.Key)
+	case structs.KeyringUse:
+		serfResp, err = mgr.UseKey(args.Key)
+	case structs.KeyringRemove:
+		serfResp, err = mgr.RemoveKey(args.Key)
+	}
+
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+
+	reply.Responses = append(reply.Responses, &structs.KeyringResponse{
+		WAN:        wan,
+		Datacenter: m.srv.config.Datacenter,
+		Messages:   serfResp.Messages,
+		Keys:       serfResp.Keys,
+		NumNodes:   serfResp.NumNodes,
+		Error:      errStr,
+	})
+}

consul/internal_endpoint_test.go

@@ -1,6 +1,8 @@
 package consul
 
 import (
+	"encoding/base64"
+	"fmt"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/consul/testutil"
 	"os"
@@ -150,3 +152,89 @@ func TestInternal_NodeDump(t *testing.T) {
 		t.Fatalf("missing foo or bar")
 	}
 }
+
+func TestInternal_KeyringOperation(t *testing.T) {
+	key1 := "H1dfkSZOVnP/JUnaBfTzXg=="
+	keyBytes1, err := base64.StdEncoding.DecodeString(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.SerfWANConfig.MemberlistConfig.SecretKey = keyBytes1
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+
+	var out structs.KeyringResponses
+	req := structs.KeyringRequest{
+		Operation:  structs.KeyringList,
+		Datacenter: "dc1",
+	}
+	if err := client.Call("Internal.KeyringOperation", &req, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Two responses (local lan/wan pools) from single-node cluster
+	if len(out.Responses) != 2 {
+		t.Fatalf("bad: %#v", out)
+	}
+	if _, ok := out.Responses[0].Keys[key1]; !ok {
+		t.Fatalf("bad: %#v", out)
+	}
+	wanResp, lanResp := 0, 0
+	for _, resp := range out.Responses {
+		if resp.WAN {
+			wanResp++
+		} else {
+			lanResp++
+		}
+	}
+	if lanResp != 1 || wanResp != 1 {
+		t.Fatalf("should have one lan and one wan response")
+	}
+
+	// Start a second agent to test cross-dc queries
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.SerfWANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.Datacenter = "dc2"
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	// Try to join
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfWANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinWAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	var out2 structs.KeyringResponses
+	req2 := structs.KeyringRequest{
+		Operation: structs.KeyringList,
+	}
+	if err := client.Call("Internal.KeyringOperation", &req2, &out2); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// 3 responses (one from each DC LAN, one from WAN) in two-node cluster
+	if len(out2.Responses) != 3 {
+		t.Fatalf("bad: %#v", out)
+	}
+	wanResp, lanResp = 0, 0
+	for _, resp := range out2.Responses {
+		if resp.WAN {
+			wanResp++
+		} else {
+			lanResp++
+		}
+	}
+	if lanResp != 2 || wanResp != 1 {
+		t.Fatalf("should have two lan and one wan response")
+	}
+}

consul/kvs_endpoint_test.go

@@ -510,7 +510,7 @@ func TestKVS_Apply_LockDelay(t *testing.T) {
 	// Create and invalidate a session with a lock
 	state := s1.fsm.State()
 	if err := state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	session := &structs.Session{
 		ID:        generateUUID(),

consul/leader.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/armon/go-metrics"
@@ -55,14 +56,14 @@ func (s *Server) leaderLoop(stopCh chan struct{}) {
 		s.logger.Printf("[WARN] consul: failed to broadcast new leader event: %v", err)
 	}
 
-	// Setup ACLs if we are the leader and need to
-	if err := s.initializeACL(); err != nil {
-		s.logger.Printf("[ERR] consul: ACL initialization failed: %v", err)
-	}
+	// Clear the session timers on either shutdown or step down, since we
+	// are no longer responsible for session expirations.
+	defer s.clearAllSessionTimers()
 
 	// Reconcile channel is only used once initial reconcile
 	// has succeeded
 	var reconcileCh chan serf.Member
+	establishedLeader := false
 
 RECONCILE:
 	// Setup a reconciliation timer
@@ -78,6 +79,16 @@ RECONCILE:
 	}
 	metrics.MeasureSince([]string{"consul", "leader", "barrier"}, start)
 
+	// Check if we need to handle initial leadership actions
+	if !establishedLeader {
+		if err := s.establishLeadership(); err != nil {
+			s.logger.Printf("[ERR] consul: failed to establish leadership: %v",
+				err)
+			goto WAIT
+		}
+		establishedLeader = true
+	}
+
 	// Reconcile any missing data
 	if err := s.reconcile(); err != nil {
 		s.logger.Printf("[ERR] consul: failed to reconcile: %v", err)
@@ -105,6 +116,34 @@ WAIT:
 	}
 }
 
+// establishLeadership is invoked once we become leader and are able
+// to invoke an initial barrier. The barrier is used to ensure any
+// previously inflight transactions have been commited and that our
+// state is up-to-date.
+func (s *Server) establishLeadership() error {
+	// Setup ACLs if we are the leader and need to
+	if err := s.initializeACL(); err != nil {
+		s.logger.Printf("[ERR] consul: ACL initialization failed: %v", err)
+		return err
+	}
+
+	// Setup the session timers. This is done both when starting up or when
+	// a leader fail over happens. Since the timers are maintained by the leader
+	// node along, effectively this means all the timers are renewed at the
+	// time of failover. The TTL contract is that the session will not be expired
+	// before the TTL, so expiring it later is allowable.
+	//
+	// This MUST be done after the initial barrier to ensure the latest Sessions
+	// are available to be initialized. Otherwise initialization may use stale
+	// data.
+	if err := s.initializeSessionTimers(); err != nil {
+		s.logger.Printf("[ERR] consul: Session Timers initialization failed: %v",
+			err)
+		return err
+	}
+	return nil
+}
+
 // initializeACL is used to setup the ACLs if we are the leader
 // and need to do this.
 func (s *Server) initializeACL() error {
@@ -265,6 +304,11 @@ func (s *Server) reconcileMember(member serf.Member) error {
 	if err != nil {
 		s.logger.Printf("[ERR] consul: failed to reconcile member: %v: %v",
 			member, err)
+
+		// Permission denied should not bubble up
+		if strings.Contains(err.Error(), permissionDenied) {
+			return nil
+		}
 		return err
 	}
 	return nil
@@ -344,6 +388,7 @@ AFTER_CHECK:
 			Status:  structs.HealthPassing,
 			Output:  SerfCheckAliveOutput,
 		},
+		WriteRequest: structs.WriteRequest{Token: s.config.ACLToken},
 	}
 	var out struct{}
 	return s.endpoints.Catalog.Register(&req, &out)
@@ -379,6 +424,7 @@ func (s *Server) handleFailedMember(member serf.Member) error {
 			Status:  structs.HealthCritical,
 			Output:  SerfCheckFailedOutput,
 		},
+		WriteRequest: structs.WriteRequest{Token: s.config.ACLToken},
 	}
 	var out struct{}
 	return s.endpoints.Catalog.Register(&req, &out)

consul/leader_test.go

@@ -370,6 +370,9 @@ func TestLeader_LeftLeader(t *testing.T) {
 			break
 		}
 	}
+	if leader == nil {
+		t.Fatalf("Should have a leader")
+	}
 	leader.Leave()
 	leader.Shutdown()
 	time.Sleep(100 * time.Millisecond)
@@ -390,10 +393,12 @@ func TestLeader_LeftLeader(t *testing.T) {
 
 	// Verify the old leader is deregistered
 	state := remain.fsm.State()
-	_, found, _ := state.GetNode(leader.config.NodeName)
-	if found {
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ := state.GetNode(leader.config.NodeName)
+		return !found, nil
+	}, func(err error) {
 		t.Fatalf("leader should be deregistered")
-	}
+	})
 }
 
 func TestLeader_MultiBootstrap(t *testing.T) {

consul/mdb_table_test.go

@@ -3,7 +3,7 @@ package consul
 import (
 	"bytes"
 	"github.com/armon/gomdb"
-	"github.com/ugorji/go/codec"
+	"github.com/hashicorp/go-msgpack/codec"
 	"io/ioutil"
 	"os"
 	"reflect"

consul/pool.go

@@ -11,9 +11,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/hashicorp/consul/tlsutil"
+	"github.com/hashicorp/go-msgpack/codec"
 	"github.com/hashicorp/yamux"
 	"github.com/inconshreveable/muxado"
-	"github.com/ugorji/go/codec"
 )
 
 // msgpackHandle is a shared handle for encoding/decoding of RPC messages
@@ -113,7 +114,7 @@ func (c *Conn) returnClient(client *StreamClient) {
 // Consul servers. This is used to reduce the latency of
 // RPC requests between servers. It is only used to pool
 // connections in the rpcConsul mode. Raft connections
-// are pooled seperately.
+// are pooled separately.
 type ConnPool struct {
 	sync.Mutex
 
@@ -222,7 +223,7 @@ func (p *ConnPool) getNewConn(addr net.Addr, version int) (*Conn, error) {
 		}
 
 		// Wrap the connection in a TLS client
-		tlsConn, err := wrapTLSClient(conn, p.tlsConfig)
+		tlsConn, err := tlsutil.WrapTLSClient(conn, p.tlsConfig)
 		if err != nil {
 			conn.Close()
 			return nil, err
@@ -258,27 +259,28 @@ func (p *ConnPool) getNewConn(addr net.Addr, version int) (*Conn, error) {
 	}
 
 	// Wrap the connection
-	c := &Conn{
-		refCount: 1,
-		addr:     addr,
-		session:  session,
-		clients:  list.New(),
-		lastUsed: time.Now(),
-		version:  version,
-		pool:     p,
-	}
+	var c *Conn
 
 	// Track this connection, handle potential race condition
 	p.Lock()
+	defer p.Unlock()
+
 	if existing := p.pool[addr.String()]; existing != nil {
-		c.Close()
-		p.Unlock()
-		return existing, nil
+		c = existing
 	} else {
+		c = &Conn{
+			refCount: 1,
+			addr:     addr,
+			session:  session,
+			clients:  list.New(),
+			lastUsed: time.Now(),
+			version:  version,
+			pool:     p,
+		}
 		p.pool[addr.String()] = c
-		p.Unlock()
-		return c, nil
 	}
+
+	return c, nil
 }
 
 // clearConn is used to clear any cached connection, potentially in response to an erro
@@ -357,12 +359,12 @@ func (p *ConnPool) RPC(addr net.Addr, version int, method string, args interface
 
 // Reap is used to close conns open over maxTime
 func (p *ConnPool) reap() {
-	for !p.shutdown {
+	for {
 		// Sleep for a while
 		select {
-		case <-time.After(time.Second):
 		case <-p.shutdownCh:
 			return
+		case <-time.After(time.Second):
 		}
 
 		// Reap all old conns

consul/raft_rpc.go

@@ -3,6 +3,7 @@ package consul
 import (
 	"crypto/tls"
 	"fmt"
+	"github.com/hashicorp/consul/tlsutil"
 	"net"
 	"sync"
 	"time"
@@ -94,7 +95,7 @@ func (l *RaftLayer) Dial(address string, timeout time.Duration) (net.Conn, error
 		}
 
 		// Wrap the connection in a TLS client
-		conn, err = wrapTLSClient(conn, l.tlsConfig)
+		conn, err = tlsutil.WrapTLSClient(conn, l.tlsConfig)
 		if err != nil {
 			return nil, err
 		}

consul/rpc.go

@@ -3,16 +3,17 @@ package consul
 import (
 	"crypto/tls"
 	"fmt"
-	"github.com/armon/go-metrics"
-	"github.com/hashicorp/consul/consul/structs"
-	"github.com/hashicorp/yamux"
-	"github.com/inconshreveable/muxado"
-	"github.com/ugorji/go/codec"
 	"io"
 	"math/rand"
 	"net"
 	"strings"
 	"time"
+
+	"github.com/armon/go-metrics"
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/go-msgpack/codec"
+	"github.com/hashicorp/yamux"
+	"github.com/inconshreveable/muxado"
 )
 
 type RPCType byte
@@ -149,7 +150,13 @@ func (s *Server) handleMultiplexV2(conn net.Conn) {
 func (s *Server) handleConsulConn(conn net.Conn) {
 	defer conn.Close()
 	rpcCodec := codec.GoRpc.ServerCodec(conn, msgpackHandle)
-	for !s.shutdown {
+	for {
+		select {
+		case <-s.shutdownCh:
+			return
+		default:
+		}
+
 		if err := s.rpcServer.ServeRequest(rpcCodec); err != nil {
 			if err != io.EOF && !strings.Contains(err.Error(), "closed") {
 				s.logger.Printf("[ERR] consul.rpc: RPC error: %v (%v)", err, conn)
@@ -223,6 +230,40 @@ func (s *Server) forwardDC(method, dc string, args interface{}, reply interface{
 	return s.connPool.RPC(server.Addr, server.Version, method, args, reply)
 }
 
+// globalRPC is used to forward an RPC request to one server in each datacenter.
+// This will only error for RPC-related errors. Otherwise, application-level
+// errors can be sent in the response objects.
+func (s *Server) globalRPC(method string, args interface{},
+	reply structs.CompoundResponse) error {
+
+	errorCh := make(chan error)
+	respCh := make(chan interface{})
+
+	// Make a new request into each datacenter
+	for dc, _ := range s.remoteConsuls {
+		go func(dc string) {
+			rr := reply.New()
+			if err := s.forwardDC(method, dc, args, &rr); err != nil {
+				errorCh <- err
+				return
+			}
+			respCh <- rr
+		}(dc)
+	}
+
+	replies, total := 0, len(s.remoteConsuls)
+	for replies < total {
+		select {
+		case err := <-errorCh:
+			return err
+		case rr := <-respCh:
+			reply.Add(rr)
+			replies++
+		}
+	}
+	return nil
+}
+
 // raftApply is used to encode a message, run it through raft, and return
 // the FSM response along with any errors
 func (s *Server) raftApply(t structs.MessageType, msg interface{}) (interface{}, error) {

consul/serf.go

@@ -147,7 +147,7 @@ func (s *Server) nodeJoin(me serf.MemberEvent, wan bool) {
 		ok, parts := isConsulServer(m)
 		if !ok {
 			if wan {
-				s.logger.Printf("[WARN] consul: non-server in WAN pool: %s %s", m.Name)
+				s.logger.Printf("[WARN] consul: non-server in WAN pool: %s", m.Name)
 			}
 			continue
 		}

consul/server.go

@@ -128,6 +128,12 @@ type Server struct {
 	// which SHOULD only consist of Consul servers
 	serfWAN *serf.Serf
 
+	// sessionTimers track the expiration time of each Session that has
+	// a TTL. On expiration, a SessionDestroy event will occur, and
+	// destroy the session via standard session destory processing
+	sessionTimers     map[string]*time.Timer
+	sessionTimersLock sync.Mutex
+
 	shutdown     bool
 	shutdownCh   chan struct{}
 	shutdownLock sync.Mutex
@@ -168,13 +174,14 @@ func NewServer(config *Config) (*Server, error) {
 	}
 
 	// Create the tlsConfig for outgoing connections
-	tlsConfig, err := config.OutgoingTLSConfig()
+	tlsConf := config.tlsConfig()
+	tlsConfig, err := tlsConf.OutgoingTLSConfig()
 	if err != nil {
 		return nil, err
 	}
 
 	// Get the incoming tls config
-	incomingTLS, err := config.IncomingTLSConfig()
+	incomingTLS, err := tlsConf.IncomingTLSConfig()
 	if err != nil {
 		return nil, err
 	}
@@ -550,6 +557,21 @@ func (s *Server) IsLeader() bool {
 	return s.raft.State() == raft.Leader
 }
 
+// KeyManagerLAN returns the LAN Serf keyring manager
+func (s *Server) KeyManagerLAN() *serf.KeyManager {
+	return s.serfLAN.KeyManager()
+}
+
+// KeyManagerWAN returns the WAN Serf keyring manager
+func (s *Server) KeyManagerWAN() *serf.KeyManager {
+	return s.serfWAN.KeyManager()
+}
+
+// Encrypted determines if gossip is encrypted
+func (s *Server) Encrypted() bool {
+	return s.serfLAN.EncryptionEnabled() && s.serfWAN.EncryptionEnabled()
+}
+
 // inmemCodec is used to do an RPC call without going over a network
 type inmemCodec struct {
 	method string

consul/server_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -255,10 +256,12 @@ func TestServer_Leave(t *testing.T) {
 	}
 
 	// Should lose a peer
-	p1, _ = s1.raftPeers.Peers()
-	if len(p1) != 1 {
+	testutil.WaitForResult(func() (bool, error) {
+		p1, _ = s1.raftPeers.Peers()
+		return len(p1) == 1, nil
+	}, func(err error) {
 		t.Fatalf("should have 1 peer: %v", p1)
-	}
+	})
 }
 
 func TestServer_RPC(t *testing.T) {
@@ -471,5 +474,56 @@ func TestServer_BadExpect(t *testing.T) {
 	}, func(err error) {
 		t.Fatalf("should have 0 peers: %v", err)
 	})
-
+}
+
+type fakeGlobalResp struct{}
+
+func (r *fakeGlobalResp) Add(interface{}) {
+	return
+}
+
+func (r *fakeGlobalResp) New() interface{} {
+	return struct{}{}
+}
+
+func TestServer_globalRPCErrors(t *testing.T) {
+	dir1, s1 := testServerDC(t, "dc1")
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s1.remoteConsuls) == 1, nil
+	}, func(err error) {
+		t.Fatalf("Server did not join LAN successfully")
+	})
+
+	// Check that an error from a remote DC is returned
+	err := s1.globalRPC("Bad.Method", nil, &fakeGlobalResp{})
+	if err == nil {
+		t.Fatalf("should have errored")
+	}
+	if !strings.Contains(err.Error(), "Bad.Method") {
+		t.Fatalf("unexpcted error: %s", err)
+	}
+}
+
+func TestServer_Encrypted(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	key := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = key
+		c.SerfWANConfig.MemberlistConfig.SecretKey = key
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	if s1.Encrypted() {
+		t.Fatalf("should not be encrypted")
+	}
+	if !s2.Encrypted() {
+		t.Fatalf("should be encrypted")
+	}
 }

consul/session_endpoint.go

@@ -29,6 +29,30 @@ func (s *Session) Apply(args *structs.SessionRequest, reply *string) error {
 		return fmt.Errorf("Must provide Node")
 	}
 
+	// Ensure that the specified behavior is allowed
+	switch args.Session.Behavior {
+	case "":
+		// Default behavior to Release for backwards compatibility
+		args.Session.Behavior = structs.SessionKeysRelease
+	case structs.SessionKeysRelease:
+	case structs.SessionKeysDelete:
+	default:
+		return fmt.Errorf("Invalid Behavior setting '%s'", args.Session.Behavior)
+	}
+
+	// Ensure the Session TTL is valid if provided
+	if args.Session.TTL != "" {
+		ttl, err := time.ParseDuration(args.Session.TTL)
+		if err != nil {
+			return fmt.Errorf("Session TTL '%s' invalid: %v", args.Session.TTL, err)
+		}
+
+		if ttl != 0 && (ttl < structs.SessionTTLMin || ttl > structs.SessionTTLMax) {
+			return fmt.Errorf("Invalid Session TTL '%d', must be between [%v=%v]",
+				ttl, structs.SessionTTLMin, structs.SessionTTLMax)
+		}
+	}
+
 	// If this is a create, we must generate the Session ID. This must
 	// be done prior to appending to the raft log, because the ID is not
 	// deterministic. Once the entry is in the log, the state update MUST
@@ -55,6 +79,16 @@ func (s *Session) Apply(args *structs.SessionRequest, reply *string) error {
 		s.srv.logger.Printf("[ERR] consul.session: Apply failed: %v", err)
 		return err
 	}
+
+	if args.Op == structs.SessionCreate && args.Session.TTL != "" {
+		// If we created a session with a TTL, reset the expiration timer
+		s.srv.resetSessionTimer(args.Session.ID, &args.Session)
+	} else if args.Op == structs.SessionDestroy {
+		// If we destroyed a session, it might potentially have a TTL,
+		// and we need to clear the timer
+		s.srv.clearSessionTimer(args.Session.ID)
+	}
+
 	if respErr, ok := resp.(error); ok {
 		return respErr
 	}
@@ -125,3 +159,29 @@ func (s *Session) NodeSessions(args *structs.NodeSpecificRequest,
 			return err
 		})
 }
+
+// Renew is used to renew the TTL on a single session
+func (s *Session) Renew(args *structs.SessionSpecificRequest,
+	reply *structs.IndexedSessions) error {
+	if done, err := s.srv.forward("Session.Renew", args, args, reply); done {
+		return err
+	}
+
+	// Get the session, from local state
+	state := s.srv.fsm.State()
+	index, session, err := state.SessionGet(args.Session)
+	if err != nil {
+		return err
+	}
+
+	// Reset the session TTL timer
+	reply.Index = index
+	if session != nil {
+		reply.Sessions = structs.Sessions{session}
+		if err := s.srv.resetSessionTimer(args.Session, session); err != nil {
+			s.srv.logger.Printf("[ERR] consul.session: Session renew failed: %v", err)
+			return err
+		}
+	}
+	return nil
+}

consul/session_endpoint_test.go

@@ -1,10 +1,12 @@
 package consul
 
 import (
-	"github.com/hashicorp/consul/consul/structs"
-	"github.com/hashicorp/consul/testutil"
 	"os"
 	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/testutil"
 )
 
 func TestSessionEndpoint_Apply(t *testing.T) {
@@ -66,6 +68,69 @@ func TestSessionEndpoint_Apply(t *testing.T) {
 	}
 }
 
+func TestSessionEndpoint_DeleteApply(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+
+	// Just add a node
+	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+
+	arg := structs.SessionRequest{
+		Datacenter: "dc1",
+		Op:         structs.SessionCreate,
+		Session: structs.Session{
+			Node:     "foo",
+			Name:     "my-session",
+			Behavior: structs.SessionKeysDelete,
+		},
+	}
+	var out string
+	if err := client.Call("Session.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	id := out
+
+	// Verify
+	state := s1.fsm.State()
+	_, s, err := state.SessionGet(out)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if s == nil {
+		t.Fatalf("should not be nil")
+	}
+	if s.Node != "foo" {
+		t.Fatalf("bad: %v", s)
+	}
+	if s.Name != "my-session" {
+		t.Fatalf("bad: %v", s)
+	}
+	if s.Behavior != structs.SessionKeysDelete {
+		t.Fatalf("bad: %v", s)
+	}
+
+	// Do a delete
+	arg.Op = structs.SessionDestroy
+	arg.Session.ID = out
+	if err := client.Call("Session.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Verify
+	_, s, err = state.SessionGet(id)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if s != nil {
+		t.Fatalf("bad: %v", s)
+	}
+}
+
 func TestSessionEndpoint_Get(t *testing.T) {
 	dir1, s1 := testServer(t)
 	defer os.RemoveAll(dir1)
@@ -160,6 +225,207 @@ func TestSessionEndpoint_List(t *testing.T) {
 	}
 }
 
+func TestSessionEndpoint_ApplyTimers(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+
+	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	arg := structs.SessionRequest{
+		Datacenter: "dc1",
+		Op:         structs.SessionCreate,
+		Session: structs.Session{
+			Node: "foo",
+			TTL:  "10s",
+		},
+	}
+	var out string
+	if err := client.Call("Session.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the session map
+	if _, ok := s1.sessionTimers[out]; !ok {
+		t.Fatalf("missing session timer")
+	}
+
+	// Destroy the session
+	arg.Op = structs.SessionDestroy
+	arg.Session.ID = out
+	if err := client.Call("Session.Apply", &arg, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the session map
+	if _, ok := s1.sessionTimers[out]; ok {
+		t.Fatalf("session timer exists")
+	}
+}
+
+func TestSessionEndpoint_Renew(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+	TTL := "10s" // the minimum allowed ttl
+	ttl := 10 * time.Second
+
+	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	ids := []string{}
+	for i := 0; i < 5; i++ {
+		arg := structs.SessionRequest{
+			Datacenter: "dc1",
+			Op:         structs.SessionCreate,
+			Session: structs.Session{
+				Node: "foo",
+				TTL:  TTL,
+			},
+		}
+		var out string
+		if err := client.Call("Session.Apply", &arg, &out); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		ids = append(ids, out)
+	}
+
+	// Verify the timer map is setup
+	if len(s1.sessionTimers) != 5 {
+		t.Fatalf("missing session timers")
+	}
+
+	getR := structs.DCSpecificRequest{
+		Datacenter: "dc1",
+	}
+
+	var sessions structs.IndexedSessions
+	if err := client.Call("Session.List", &getR, &sessions); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if sessions.Index == 0 {
+		t.Fatalf("Bad: %v", sessions)
+	}
+	if len(sessions.Sessions) != 5 {
+		t.Fatalf("Bad: %v", sessions.Sessions)
+	}
+	for i := 0; i < len(sessions.Sessions); i++ {
+		s := sessions.Sessions[i]
+		if !strContains(ids, s.ID) {
+			t.Fatalf("bad: %v", s)
+		}
+		if s.Node != "foo" {
+			t.Fatalf("bad: %v", s)
+		}
+		if s.TTL != TTL {
+			t.Fatalf("bad session TTL: %s %v", s.TTL, s)
+		}
+		t.Logf("Created session '%s'", s.ID)
+	}
+
+	// Sleep for time shorter than internal destroy ttl
+	time.Sleep(ttl * structs.SessionTTLMultiplier / 2)
+
+	// renew 3 out of 5 sessions
+	for i := 0; i < 3; i++ {
+		renewR := structs.SessionSpecificRequest{
+			Datacenter: "dc1",
+			Session:    ids[i],
+		}
+		var session structs.IndexedSessions
+		if err := client.Call("Session.Renew", &renewR, &session); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if session.Index == 0 {
+			t.Fatalf("Bad: %v", session)
+		}
+		if len(session.Sessions) != 1 {
+			t.Fatalf("Bad: %v", session.Sessions)
+		}
+
+		s := session.Sessions[0]
+		if !strContains(ids, s.ID) {
+			t.Fatalf("bad: %v", s)
+		}
+		if s.Node != "foo" {
+			t.Fatalf("bad: %v", s)
+		}
+
+		t.Logf("Renewed session '%s'", s.ID)
+	}
+
+	// now sleep for 2/3 the internal destroy TTL time for renewed sessions
+	// which is more than the internal destroy TTL time for the non-renewed sessions
+	time.Sleep((ttl * structs.SessionTTLMultiplier) * 2.0 / 3.0)
+
+	var sessionsL1 structs.IndexedSessions
+	if err := client.Call("Session.List", &getR, &sessionsL1); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if sessionsL1.Index == 0 {
+		t.Fatalf("Bad: %v", sessionsL1)
+	}
+
+	t.Logf("Expect 2 sessions to be destroyed")
+
+	for i := 0; i < len(sessionsL1.Sessions); i++ {
+		s := sessionsL1.Sessions[i]
+		if !strContains(ids, s.ID) {
+			t.Fatalf("bad: %v", s)
+		}
+		if s.Node != "foo" {
+			t.Fatalf("bad: %v", s)
+		}
+		if s.TTL != TTL {
+			t.Fatalf("bad: %v", s)
+		}
+		if i > 2 {
+			t.Errorf("session '%s' should be destroyed", s.ID)
+		}
+	}
+
+	if len(sessionsL1.Sessions) != 3 {
+		t.Fatalf("Bad: %v", sessionsL1.Sessions)
+	}
+
+	// now sleep again for ttl*2 - no sessions should still be alive
+	time.Sleep(ttl * structs.SessionTTLMultiplier)
+
+	var sessionsL2 structs.IndexedSessions
+	if err := client.Call("Session.List", &getR, &sessionsL2); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if sessionsL2.Index == 0 {
+		t.Fatalf("Bad: %v", sessionsL2)
+	}
+	if len(sessionsL2.Sessions) != 0 {
+		for i := 0; i < len(sessionsL2.Sessions); i++ {
+			s := sessionsL2.Sessions[i]
+			if !strContains(ids, s.ID) {
+				t.Fatalf("bad: %v", s)
+			}
+			if s.Node != "foo" {
+				t.Fatalf("bad: %v", s)
+			}
+			if s.TTL != TTL {
+				t.Fatalf("bad: %v", s)
+			}
+			t.Errorf("session '%s' should be destroyed", s.ID)
+		}
+
+		t.Fatalf("Bad: %v", sessionsL2.Sessions)
+	}
+}
+
 func TestSessionEndpoint_NodeSessions(t *testing.T) {
 	dir1, s1 := testServer(t)
 	defer os.RemoveAll(dir1)

consul/session_ttl.go

@@ -0,0 +1,144 @@
+package consul
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/consul/consul/structs"
+)
+
+// initializeSessionTimers is used when a leader is newly electd to create
+// a new map to track session expiration and to reset all the timers from
+// the previously known set of timers.
+func (s *Server) initializeSessionTimers() error {
+	// Scan all sessions and reset their timer
+	state := s.fsm.State()
+	_, sessions, err := state.SessionList()
+	if err != nil {
+		return err
+	}
+	for _, session := range sessions {
+		if err := s.resetSessionTimer(session.ID, session); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// resetSessionTimer is used to renew the TTL of a session.
+// This can be used for new sessions and existing ones. A session
+// will be faulted in if not given.
+func (s *Server) resetSessionTimer(id string, session *structs.Session) error {
+	// Fault the session in if not given
+	if session == nil {
+		state := s.fsm.State()
+		_, s, err := state.SessionGet(id)
+		if err != nil {
+			return err
+		}
+		if s == nil {
+			return fmt.Errorf("Session '%s' not found", id)
+		}
+		session = s
+	}
+
+	// Bail if the session has no TTL, fast-path some common inputs
+	switch session.TTL {
+	case "", "0", "0s", "0m", "0h":
+		return nil
+	}
+
+	// Parse the TTL, and skip if zero time
+	ttl, err := time.ParseDuration(session.TTL)
+	if err != nil {
+		return fmt.Errorf("Invalid Session TTL '%s': %v", session.TTL, err)
+	}
+	if ttl == 0 {
+		return nil
+	}
+
+	// Reset the session timer
+	s.sessionTimersLock.Lock()
+	defer s.sessionTimersLock.Unlock()
+	s.resetSessionTimerLocked(id, ttl)
+	return nil
+}
+
+// resetSessionTimerLocked is used to reset a session timer
+// assuming the sessionTimerLock is already held
+func (s *Server) resetSessionTimerLocked(id string, ttl time.Duration) {
+	// Ensure a timer map exists
+	if s.sessionTimers == nil {
+		s.sessionTimers = make(map[string]*time.Timer)
+	}
+
+	// Adjust the given TTL by the TTL multiplier. This is done
+	// to give a client a grace period and to compensate for network
+	// and processing delays. The contract is that a session is not expired
+	// before the TTL, but there is no explicit promise about the upper
+	// bound so this is allowable.
+	ttl = ttl * structs.SessionTTLMultiplier
+
+	// Renew the session timer if it exists
+	if timer, ok := s.sessionTimers[id]; ok {
+		timer.Reset(ttl)
+		return
+	}
+
+	// Create a new timer to track expiration of thi ssession
+	timer := time.AfterFunc(ttl, func() {
+		s.invalidateSession(id)
+	})
+	s.sessionTimers[id] = timer
+}
+
+// invalidateSession is invoked when a session TTL is reached and we
+// need to invalidate the session.
+func (s *Server) invalidateSession(id string) {
+	// Clear the session timer
+	s.sessionTimersLock.Lock()
+	delete(s.sessionTimers, id)
+	s.sessionTimersLock.Unlock()
+
+	// Create a session destroy request
+	args := structs.SessionRequest{
+		Datacenter: s.config.Datacenter,
+		Op:         structs.SessionDestroy,
+		Session: structs.Session{
+			ID: id,
+		},
+	}
+	s.logger.Printf("[DEBUG] consul.state: Session %s TTL expired", id)
+
+	// Apply the update to destroy the session
+	if _, err := s.raftApply(structs.SessionRequestType, args); err != nil {
+		s.logger.Printf("[ERR] consul.session: Invalidation failed: %v", err)
+	}
+}
+
+// clearSessionTimer is used to clear the session time for
+// a single session. This is used when a session is destroyed
+// explicitly and no longer needed.
+func (s *Server) clearSessionTimer(id string) error {
+	s.sessionTimersLock.Lock()
+	defer s.sessionTimersLock.Unlock()
+
+	if timer, ok := s.sessionTimers[id]; ok {
+		timer.Stop()
+		delete(s.sessionTimers, id)
+	}
+	return nil
+}
+
+// clearAllSessionTimers is used when a leader is stepping
+// down and we no longer need to track any session timers.
+func (s *Server) clearAllSessionTimers() error {
+	s.sessionTimersLock.Lock()
+	defer s.sessionTimersLock.Unlock()
+
+	for _, t := range s.sessionTimers {
+		t.Stop()
+	}
+	s.sessionTimers = nil
+	return nil
+}

consul/session_ttl_test.go

@@ -0,0 +1,370 @@
+package consul
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/testutil"
+)
+
+func TestInitializeSessionTimers(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	state := s1.fsm.State()
+	state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	session := &structs.Session{
+		ID:   generateUUID(),
+		Node: "foo",
+		TTL:  "10s",
+	}
+	if err := state.SessionCreate(100, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Reset the session timers
+	err := s1.initializeSessionTimers()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check that we have a timer
+	_, ok := s1.sessionTimers[session.ID]
+	if !ok {
+		t.Fatalf("missing session timer")
+	}
+}
+
+func TestResetSessionTimer_Fault(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	// Should not exist
+	err := s1.resetSessionTimer("nope", nil)
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Create a session
+	state := s1.fsm.State()
+	state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	session := &structs.Session{
+		ID:   generateUUID(),
+		Node: "foo",
+		TTL:  "10s",
+	}
+	if err := state.SessionCreate(100, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Reset the session timer
+	err = s1.resetSessionTimer(session.ID, nil)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check that we have a timer
+	_, ok := s1.sessionTimers[session.ID]
+	if !ok {
+		t.Fatalf("missing session timer")
+	}
+}
+
+func TestResetSessionTimer_NoTTL(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	// Create a session
+	state := s1.fsm.State()
+	state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	session := &structs.Session{
+		ID:   generateUUID(),
+		Node: "foo",
+		TTL:  "0000s",
+	}
+	if err := state.SessionCreate(100, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Reset the session timer
+	err := s1.resetSessionTimer(session.ID, session)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check that we have a timer
+	_, ok := s1.sessionTimers[session.ID]
+	if ok {
+		t.Fatalf("should not have session timer")
+	}
+}
+
+func TestResetSessionTimer_InvalidTTL(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	// Create a session
+	session := &structs.Session{
+		ID:   generateUUID(),
+		Node: "foo",
+		TTL:  "foo",
+	}
+
+	// Reset the session timer
+	err := s1.resetSessionTimer(session.ID, session)
+	if err == nil || !strings.Contains(err.Error(), "Invalid Session TTL") {
+		t.Fatalf("err: %v", err)
+	}
+}
+
+func TestResetSessionTimerLocked(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	s1.sessionTimersLock.Lock()
+	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
+	s1.sessionTimersLock.Unlock()
+
+	if _, ok := s1.sessionTimers["foo"]; !ok {
+		t.Fatalf("missing timer")
+	}
+
+	time.Sleep(10 * time.Millisecond * structs.SessionTTLMultiplier)
+
+	if _, ok := s1.sessionTimers["foo"]; ok {
+		t.Fatalf("timer should be gone")
+	}
+}
+
+func TestResetSessionTimerLocked_Renew(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	s1.sessionTimersLock.Lock()
+	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
+	s1.sessionTimersLock.Unlock()
+
+	if _, ok := s1.sessionTimers["foo"]; !ok {
+		t.Fatalf("missing timer")
+	}
+
+	time.Sleep(5 * time.Millisecond)
+
+	// Renew the session
+	s1.sessionTimersLock.Lock()
+	renew := time.Now()
+	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
+	s1.sessionTimersLock.Unlock()
+
+	// Watch for invalidation
+	for time.Now().Sub(renew) < 20*time.Millisecond {
+		s1.sessionTimersLock.Lock()
+		_, ok := s1.sessionTimers["foo"]
+		s1.sessionTimersLock.Unlock()
+		if !ok {
+			end := time.Now()
+			if end.Sub(renew) < 5*time.Millisecond {
+				t.Fatalf("early invalidate")
+			}
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("should have expired")
+}
+
+func TestInvalidateSession(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	// Create a session
+	state := s1.fsm.State()
+	state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
+	session := &structs.Session{
+		ID:   generateUUID(),
+		Node: "foo",
+		TTL:  "10s",
+	}
+	if err := state.SessionCreate(100, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// This should cause a destroy
+	s1.invalidateSession(session.ID)
+
+	// Check it is gone
+	_, sess, err := state.SessionGet(session.ID)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if sess != nil {
+		t.Fatalf("should destroy session")
+	}
+}
+
+func TestClearSessionTimer(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	s1.sessionTimersLock.Lock()
+	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
+	s1.sessionTimersLock.Unlock()
+
+	err := s1.clearSessionTimer("foo")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if _, ok := s1.sessionTimers["foo"]; ok {
+		t.Fatalf("timer should be gone")
+	}
+}
+
+func TestClearAllSessionTimers(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	s1.sessionTimersLock.Lock()
+	s1.resetSessionTimerLocked("foo", 10*time.Millisecond)
+	s1.resetSessionTimerLocked("bar", 10*time.Millisecond)
+	s1.resetSessionTimerLocked("baz", 10*time.Millisecond)
+	s1.sessionTimersLock.Unlock()
+
+	err := s1.clearAllSessionTimers()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if len(s1.sessionTimers) != 0 {
+		t.Fatalf("timers should be gone")
+	}
+}
+
+func TestServer_SessionTTL_Failover(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	dir2, s2 := testServerDCBootstrap(t, "dc1", false)
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	dir3, s3 := testServerDCBootstrap(t, "dc1", false)
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+	servers := []*Server{s1, s2, s3}
+
+	// Try to join
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := s3.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	testutil.WaitForResult(func() (bool, error) {
+		peers, _ := s1.raftPeers.Peers()
+		return len(peers) == 3, nil
+	}, func(err error) {
+		t.Fatalf("should have 3 peers")
+	})
+
+	// Find the leader
+	var leader *Server
+	for _, s := range servers {
+		// Check that s.sessionTimers is empty
+		if len(s.sessionTimers) != 0 {
+			t.Fatalf("should have no sessionTimers")
+		}
+		// Find the leader too
+		if s.IsLeader() {
+			leader = s
+		}
+	}
+	if leader == nil {
+		t.Fatalf("Should have a leader")
+	}
+
+	client := rpcClient(t, leader)
+	defer client.Close()
+
+	// Register a node
+	node := structs.RegisterRequest{
+		Datacenter: s1.config.Datacenter,
+		Node:       "foo",
+		Address:    "127.0.0.1",
+	}
+	var out struct{}
+	if err := s1.RPC("Catalog.Register", &node, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Create a TTL session
+	arg := structs.SessionRequest{
+		Datacenter: "dc1",
+		Op:         structs.SessionCreate,
+		Session: structs.Session{
+			Node: "foo",
+			TTL:  "10s",
+		},
+	}
+	var id1 string
+	if err := client.Call("Session.Apply", &arg, &id1); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check that sessionTimers has the session ID
+	if _, ok := leader.sessionTimers[id1]; !ok {
+		t.Fatalf("missing session timer")
+	}
+
+	// Shutdown the leader!
+	leader.Shutdown()
+
+	// sessionTimers should be cleared on leader shutdown
+	if len(leader.sessionTimers) != 0 {
+		t.Fatalf("session timers should be empty on the shutdown leader")
+	}
+
+	// Find the new leader
+	time.Sleep(200 * time.Millisecond)
+	leader = nil
+	for _, s := range servers {
+		if s.IsLeader() {
+			leader = s
+		}
+	}
+	if leader == nil {
+		t.Fatalf("Should have a new leader")
+	}
+
+	// Ensure session timer is restored
+	if _, ok := leader.sessionTimers[id1]; !ok {
+		t.Fatalf("missing session timer")
+	}
+}

consul/state_store.go

@@ -25,6 +25,7 @@ const (
 	dbACLs                   = "acls"
 	dbMaxMapSize32bit uint64 = 128 * 1024 * 1024       // 128MB maximum size
 	dbMaxMapSize64bit uint64 = 32 * 1024 * 1024 * 1024 // 32GB maximum size
+	dbMaxReaders      uint   = 4096                    // 4K, default is 126
 )
 
 // kvMode is used internally to control which type of set
@@ -163,6 +164,12 @@ func (s *StateStore) initialize() error {
 		return err
 	}
 
+	// Increase the maximum number of concurrent readers
+	// TODO: Block transactions if we could exceed dbMaxReaders
+	if err := s.env.SetMaxReaders(dbMaxReaders); err != nil {
+		return err
+	}
+
 	// Optimize our flags for speed over safety, since the Raft log + snapshots
 	// are durable. We treat this as an ephemeral in-memory DB, since we nuke
 	// the data anyways.
@@ -1060,6 +1067,9 @@ func (s *StateStore) KVSRestore(d *structs.DirEntry) error {
 	if err := s.kvsTable.InsertTxn(tx, d); err != nil {
 		return err
 	}
+	if err := s.kvsTable.SetMaxLastIndexTxn(tx, d.ModifyIndex); err != nil {
+		return err
+	}
 	return tx.Commit()
 }
 
@@ -1096,10 +1106,18 @@ func (s *StateStore) KVSListKeys(prefix, seperator string) (uint64, []string, er
 		return 0, nil, err
 	}
 
+	// Ensure a non-zero index
+	if idx == 0 {
+		// Must provide non-zero index to prevent blocking
+		// Index 1 is impossible anyways (due to Raft internals)
+		idx = 1
+	}
+
 	// Aggregate the stream
 	stream := make(chan interface{}, 128)
 	done := make(chan struct{})
 	var keys []string
+	var maxIndex uint64
 	go func() {
 		prefixLen := len(prefix)
 		sepLen := len(seperator)
@@ -1108,6 +1126,11 @@ func (s *StateStore) KVSListKeys(prefix, seperator string) (uint64, []string, er
 			ent := raw.(*structs.DirEntry)
 			after := ent.Key[prefixLen:]
 
+			// Update the hightest index we've seen
+			if ent.ModifyIndex > maxIndex {
+				maxIndex = ent.ModifyIndex
+			}
+
 			// If there is no seperator, always accumulate
 			if sepLen == 0 {
 				keys = append(keys, ent.Key)
@@ -1131,6 +1154,11 @@ func (s *StateStore) KVSListKeys(prefix, seperator string) (uint64, []string, er
 	// Start the stream, and wait for completion
 	err = s.kvsTable.StreamTxn(stream, tx, "id_prefix", prefix)
 	<-done
+
+	// Use the maxIndex if we have any keys
+	if maxIndex != 0 {
+		idx = maxIndex
+	}
 	return idx, keys, err
 }
 
@@ -1299,6 +1327,28 @@ func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error
 		return fmt.Errorf("Missing Session ID")
 	}
 
+	switch session.Behavior {
+	case "":
+		// Default behavior is Release for backwards compatibility
+		session.Behavior = structs.SessionKeysRelease
+	case structs.SessionKeysRelease:
+	case structs.SessionKeysDelete:
+	default:
+		return fmt.Errorf("Invalid Session Behavior setting '%s'", session.Behavior)
+	}
+
+	if session.TTL != "" {
+		ttl, err := time.ParseDuration(session.TTL)
+		if err != nil {
+			return fmt.Errorf("Invalid Session TTL '%s': %v", session.TTL, err)
+		}
+
+		if ttl != 0 && (ttl < structs.SessionTTLMin || ttl > structs.SessionTTLMax) {
+			return fmt.Errorf("Invalid Session TTL '%s', must be between [%v-%v]",
+				session.TTL, structs.SessionTTLMin, structs.SessionTTLMax)
+		}
+	}
+
 	// Assign the create index
 	session.CreateIndex = index
 
@@ -1426,7 +1476,7 @@ func (s *StateStore) SessionDestroy(index uint64, id string) error {
 	}
 	defer tx.Abort()
 
-	log.Printf("[DEBUG] consul.state: Invalidating session %s due to session destroy",
+	s.logger.Printf("[DEBUG] consul.state: Invalidating session %s due to session destroy",
 		id)
 	if err := s.invalidateSession(index, tx, id); err != nil {
 		return err
@@ -1443,7 +1493,7 @@ func (s *StateStore) invalidateNode(index uint64, tx *MDBTxn, node string) error
 	}
 	for _, sess := range sessions {
 		session := sess.(*structs.Session).ID
-		log.Printf("[DEBUG] consul.state: Invalidating session %s due to node '%s' invalidation",
+		s.logger.Printf("[DEBUG] consul.state: Invalidating session %s due to node '%s' invalidation",
 			session, node)
 		if err := s.invalidateSession(index, tx, session); err != nil {
 			return err
@@ -1461,7 +1511,7 @@ func (s *StateStore) invalidateCheck(index uint64, tx *MDBTxn, node, check strin
 	}
 	for _, sc := range sessionChecks {
 		session := sc.(*sessionCheck).Session
-		log.Printf("[DEBUG] consul.state: Invalidating session %s due to check '%s' invalidation",
+		s.logger.Printf("[DEBUG] consul.state: Invalidating session %s due to check '%s' invalidation",
 			session, check)
 		if err := s.invalidateSession(index, tx, session); err != nil {
 			return err
@@ -1485,15 +1535,23 @@ func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) erro
 	}
 	session := res[0].(*structs.Session)
 
-	// Enforce the MaxLockDelay
-	delay := session.LockDelay
-	if delay > structs.MaxLockDelay {
-		delay = structs.MaxLockDelay
-	}
+	if session.Behavior == structs.SessionKeysDelete {
+		// delete the keys held by the session
+		if err := s.deleteKeys(index, tx, id); err != nil {
+			return err
+		}
 
-	// Invalidate any held locks
-	if err := s.invalidateLocks(index, tx, delay, id); err != nil {
-		return err
+	} else { // default to release
+		// Enforce the MaxLockDelay
+		delay := session.LockDelay
+		if delay > structs.MaxLockDelay {
+			delay = structs.MaxLockDelay
+		}
+
+		// Invalidate any held locks
+		if err := s.invalidateLocks(index, tx, delay, id); err != nil {
+			return err
+		}
 	}
 
 	// Nuke the session
@@ -1560,6 +1618,23 @@ func (s *StateStore) invalidateLocks(index uint64, tx *MDBTxn,
 	return nil
 }
 
+// deleteKeys is used to delete all the keys created by a session
+// within a given txn. All tables should be locked in the tx.
+func (s *StateStore) deleteKeys(index uint64, tx *MDBTxn, id string) error {
+	num, err := s.kvsTable.DeleteTxn(tx, "session", id)
+	if err != nil {
+		return err
+	}
+
+	if num > 0 {
+		if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil {
+			return err
+		}
+		tx.Defer(func() { s.watch[s.kvsTable].Notify() })
+	}
+	return nil
+}
+
 // ACLSet is used to create or update an ACL entry
 func (s *StateStore) ACLSet(index uint64, acl *structs.ACL) error {
 	// Check for an ID
@@ -1618,6 +1693,9 @@ func (s *StateStore) ACLRestore(acl *structs.ACL) error {
 	if err := s.aclTable.InsertTxn(tx, acl); err != nil {
 		return err
 	}
+	if err := s.aclTable.SetMaxLastIndexTxn(tx, acl.ModifyIndex); err != nil {
+		return err
+	}
 	return tx.Commit()
 }
 

consul/state_store_test.go

@@ -35,7 +35,7 @@ func TestEnsureRegistration(t *testing.T) {
 	}
 
 	if err := store.EnsureRegistration(13, reg); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, found, addr := store.GetNode("foo")
@@ -73,7 +73,7 @@ func TestEnsureNode(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, found, addr := store.GetNode("foo")
@@ -82,7 +82,7 @@ func TestEnsureNode(t *testing.T) {
 	}
 
 	if err := store.EnsureNode(4, structs.Node{"foo", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, found, addr = store.GetNode("foo")
@@ -99,11 +99,11 @@ func TestGetNodes(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(40, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(41, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, nodes := store.Nodes()
@@ -126,11 +126,11 @@ func BenchmarkGetNodes(b *testing.B) {
 	defer store.Close()
 
 	if err := store.EnsureNode(100, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		b.Fatalf("err: %v")
+		b.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(101, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		b.Fatalf("err: %v")
+		b.Fatalf("err: %v", err)
 	}
 
 	for i := 0; i < b.N; i++ {
@@ -259,7 +259,7 @@ func TestDeleteNodeService(t *testing.T) {
 		ServiceID: "api",
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.DeleteNodeService(14, "foo", "api"); err != nil {
@@ -329,11 +329,11 @@ func TestDeleteNode(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(20, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(21, "foo", &structs.NodeService{"api", "api", nil, 5000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	check := &structs.HealthCheck{
@@ -384,23 +384,23 @@ func TestGetServices(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(30, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(31, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(32, "foo", &structs.NodeService{"api", "api", nil, 5000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(33, "foo", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(34, "bar", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, services := store.Services()
@@ -434,31 +434,31 @@ func TestServiceNodes(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(10, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(11, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(12, "foo", &structs.NodeService{"api", "api", nil, 5000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(13, "bar", &structs.NodeService{"api", "api", nil, 5000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(14, "foo", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(15, "bar", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(16, "bar", &structs.NodeService{"db2", "db", []string{"slave"}, 8001}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, nodes := store.ServiceNodes("db")
@@ -525,23 +525,23 @@ func TestServiceTagNodes(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(15, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(16, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(17, "foo", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(18, "foo", &structs.NodeService{"db2", "db", []string{"slave"}, 8001}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(19, "bar", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, nodes := store.ServiceTagNodes("db", "master")
@@ -573,23 +573,23 @@ func TestServiceTagNodes_MultipleTags(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(15, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(16, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(17, "foo", &structs.NodeService{"db", "db", []string{"master", "v2"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(18, "foo", &structs.NodeService{"db2", "db", []string{"slave", "v2", "dev"}, 8001}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(19, "bar", &structs.NodeService{"db", "db", []string{"slave", "v2"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, nodes := store.ServiceTagNodes("db", "master")
@@ -649,23 +649,23 @@ func TestStoreSnapshot(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(8, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureNode(9, structs.Node{"bar", "127.0.0.2"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(10, "foo", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(11, "foo", &structs.NodeService{"db2", "db", []string{"slave"}, 8001}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.EnsureService(12, "bar", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	check := &structs.HealthCheck{
@@ -676,7 +676,7 @@ func TestStoreSnapshot(t *testing.T) {
 		ServiceID: "db",
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	// Add some KVS entries
@@ -703,13 +703,17 @@ func TestStoreSnapshot(t *testing.T) {
 	if ok, err := store.KVSLock(18, d); err != nil || !ok {
 		t.Fatalf("err: %v", err)
 	}
+	session = &structs.Session{ID: generateUUID(), Node: "bar", TTL: "60s"}
+	if err := store.SessionCreate(19, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
 
 	a1 := &structs.ACL{
 		ID:   generateUUID(),
 		Name: "User token",
 		Type: structs.ACLTypeClient,
 	}
-	if err := store.ACLSet(19, a1); err != nil {
+	if err := store.ACLSet(20, a1); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -718,19 +722,19 @@ func TestStoreSnapshot(t *testing.T) {
 		Name: "User token",
 		Type: structs.ACLTypeClient,
 	}
-	if err := store.ACLSet(20, a2); err != nil {
+	if err := store.ACLSet(21, a2); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Take a snapshot
 	snap, err := store.Snapshot()
 	if err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	defer snap.Close()
 
 	// Check the last nodes
-	if idx := snap.LastIndex(); idx != 20 {
+	if idx := snap.LastIndex(); idx != 21 {
 		t.Fatalf("bad: %v", idx)
 	}
 
@@ -785,15 +789,25 @@ func TestStoreSnapshot(t *testing.T) {
 		t.Fatalf("missing KVS entries!")
 	}
 
-	// Check there are 2 sessions
+	// Check there are 3 sessions
 	sessions, err := snap.SessionList()
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if len(sessions) != 2 {
+	if len(sessions) != 3 {
 		t.Fatalf("missing sessions")
 	}
 
+	ttls := 0
+	for _, session := range sessions {
+		if session.TTL != "" {
+			ttls++
+		}
+	}
+	if ttls != 1 {
+		t.Fatalf("Wrong number of sessions with TTL")
+	}
+
 	// Check for an acl
 	acls, err := snap.ACLList()
 	if err != nil {
@@ -804,13 +818,13 @@ func TestStoreSnapshot(t *testing.T) {
 	}
 
 	// Make some changes!
-	if err := store.EnsureService(21, "foo", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
+	if err := store.EnsureService(22, "foo", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if err := store.EnsureService(22, "bar", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
+	if err := store.EnsureService(23, "bar", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if err := store.EnsureNode(23, structs.Node{"baz", "127.0.0.3"}); err != nil {
+	if err := store.EnsureNode(24, structs.Node{"baz", "127.0.0.3"}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 	checkAfter := &structs.HealthCheck{
@@ -820,16 +834,16 @@ func TestStoreSnapshot(t *testing.T) {
 		Status:    structs.HealthCritical,
 		ServiceID: "db",
 	}
-	if err := store.EnsureCheck(24, checkAfter); err != nil {
+	if err := store.EnsureCheck(26, checkAfter); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
-	if err := store.KVSDelete(25, "/web/b"); err != nil {
+	if err := store.KVSDelete(26, "/web/b"); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
 	// Nuke an ACL
-	if err := store.ACLDelete(26, a1.ID); err != nil {
+	if err := store.ACLDelete(27, a1.ID); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -883,12 +897,12 @@ func TestStoreSnapshot(t *testing.T) {
 		t.Fatalf("missing KVS entries!")
 	}
 
-	// Check there are 2 sessions
+	// Check there are 3 sessions
 	sessions, err = snap.SessionList()
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if len(sessions) != 2 {
+	if len(sessions) != 3 {
 		t.Fatalf("missing sessions")
 	}
 
@@ -913,7 +927,7 @@ func TestEnsureCheck(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:      "foo",
@@ -923,7 +937,7 @@ func TestEnsureCheck(t *testing.T) {
 		ServiceID: "db1",
 	}
 	if err := store.EnsureCheck(3, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	check2 := &structs.HealthCheck{
@@ -933,7 +947,7 @@ func TestEnsureCheck(t *testing.T) {
 		Status:  structs.HealthWarning,
 	}
 	if err := store.EnsureCheck(4, check2); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, checks := store.NodeChecks("foo")
@@ -1009,7 +1023,7 @@ func TestDeleteNodeCheck(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:      "foo",
@@ -1019,7 +1033,7 @@ func TestDeleteNodeCheck(t *testing.T) {
 		ServiceID: "db1",
 	}
 	if err := store.EnsureCheck(3, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	check2 := &structs.HealthCheck{
@@ -1029,7 +1043,7 @@ func TestDeleteNodeCheck(t *testing.T) {
 		Status:  structs.HealthWarning,
 	}
 	if err := store.EnsureCheck(4, check2); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.DeleteNodeCheck(5, "foo", "db"); err != nil {
@@ -1059,7 +1073,7 @@ func TestCheckServiceNodes(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:      "foo",
@@ -1069,7 +1083,7 @@ func TestCheckServiceNodes(t *testing.T) {
 		ServiceID: "db1",
 	}
 	if err := store.EnsureCheck(3, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check = &structs.HealthCheck{
 		Node:    "foo",
@@ -1078,7 +1092,7 @@ func TestCheckServiceNodes(t *testing.T) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(4, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, nodes := store.CheckServiceNodes("db")
@@ -1140,7 +1154,7 @@ func BenchmarkCheckServiceNodes(t *testing.B) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:      "foo",
@@ -1150,7 +1164,7 @@ func BenchmarkCheckServiceNodes(t *testing.B) {
 		ServiceID: "db1",
 	}
 	if err := store.EnsureCheck(3, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check = &structs.HealthCheck{
 		Node:    "foo",
@@ -1159,7 +1173,7 @@ func BenchmarkCheckServiceNodes(t *testing.B) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(4, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	for i := 0; i < t.N; i++ {
@@ -1184,7 +1198,7 @@ func TestSS_Register_Deregister_Query(t *testing.T) {
 		nil,
 		0}
 	if err := store.EnsureService(2, "foo", srv); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	srv = &structs.NodeService{
@@ -1193,7 +1207,7 @@ func TestSS_Register_Deregister_Query(t *testing.T) {
 		nil,
 		0}
 	if err := store.EnsureService(3, "foo", srv); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	if err := store.DeleteNode(4, "foo"); err != nil {
@@ -1220,7 +1234,7 @@ func TestNodeInfo(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:      "foo",
@@ -1230,7 +1244,7 @@ func TestNodeInfo(t *testing.T) {
 		ServiceID: "db1",
 	}
 	if err := store.EnsureCheck(3, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check = &structs.HealthCheck{
 		Node:    "foo",
@@ -1239,7 +1253,7 @@ func TestNodeInfo(t *testing.T) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(4, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, dump := store.NodeInfo("foo")
@@ -1279,13 +1293,13 @@ func TestNodeDump(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(2, "foo", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureNode(3, structs.Node{"baz", "127.0.0.2"}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 	if err := store.EnsureService(4, "baz", &structs.NodeService{"db1", "db", []string{"master"}, 8000}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	idx, dump := store.NodeDump()
@@ -1554,7 +1568,7 @@ func TestKVS_ListKeys(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if idx != 0 {
+	if idx != 1 {
 		t.Fatalf("bad: %v", idx)
 	}
 	if len(keys) != 0 {
@@ -1657,6 +1671,65 @@ func TestKVS_ListKeys(t *testing.T) {
 	}
 }
 
+func TestKVS_ListKeys_Index(t *testing.T) {
+	store, err := testStateStore()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer store.Close()
+
+	// Create the entries
+	d := &structs.DirEntry{Key: "/foo/a", Flags: 42, Value: []byte("test")}
+	if err := store.KVSSet(1000, d); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	d = &structs.DirEntry{Key: "/bar/b", Flags: 42, Value: []byte("test")}
+	if err := store.KVSSet(1001, d); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	d = &structs.DirEntry{Key: "/baz/c", Flags: 42, Value: []byte("test")}
+	if err := store.KVSSet(1002, d); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	d = &structs.DirEntry{Key: "/other/d", Flags: 42, Value: []byte("test")}
+	if err := store.KVSSet(1003, d); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	idx, keys, err := store.KVSListKeys("/foo", "")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if idx != 1000 {
+		t.Fatalf("bad: %v", idx)
+	}
+	if len(keys) != 1 {
+		t.Fatalf("bad: %v", keys)
+	}
+
+	idx, keys, err = store.KVSListKeys("/ba", "")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if idx != 1002 {
+		t.Fatalf("bad: %v", idx)
+	}
+	if len(keys) != 2 {
+		t.Fatalf("bad: %v", keys)
+	}
+
+	idx, keys, err = store.KVSListKeys("/nope", "")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if idx != 1003 {
+		t.Fatalf("bad: %v", idx)
+	}
+	if len(keys) != 0 {
+		t.Fatalf("bad: %v", keys)
+	}
+}
+
 func TestKVSDeleteTree(t *testing.T) {
 	store, err := testStateStore()
 	if err != nil {
@@ -1711,7 +1784,7 @@ func TestSessionCreate(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:    "foo",
@@ -1719,7 +1792,7 @@ func TestSessionCreate(t *testing.T) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	session := &structs.Session{
@@ -1756,7 +1829,7 @@ func TestSessionCreate_Invalid(t *testing.T) {
 
 	// Check not registered
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	if err := store.SessionCreate(1000, session); err.Error() != "Missing check 'bar' registration" {
 		t.Fatalf("err: %v", err)
@@ -1785,7 +1858,7 @@ func TestSession_Lookups(t *testing.T) {
 
 	// Create a session
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	session := &structs.Session{
 		ID:     generateUUID(),
@@ -1870,7 +1943,7 @@ func TestSessionInvalidate_CriticalHealthCheck(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:    "foo",
@@ -1878,7 +1951,7 @@ func TestSessionInvalidate_CriticalHealthCheck(t *testing.T) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	session := &structs.Session{
@@ -1914,7 +1987,7 @@ func TestSessionInvalidate_DeleteHealthCheck(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	check := &structs.HealthCheck{
 		Node:    "foo",
@@ -1922,7 +1995,7 @@ func TestSessionInvalidate_DeleteHealthCheck(t *testing.T) {
 		Status:  structs.HealthPassing,
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	session := &structs.Session{
@@ -1957,7 +2030,7 @@ func TestSessionInvalidate_DeleteNode(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	session := &structs.Session{
@@ -1970,7 +2043,7 @@ func TestSessionInvalidate_DeleteNode(t *testing.T) {
 
 	// Delete the node
 	if err := store.DeleteNode(15, "foo"); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	// Lookup by ID, should be nil
@@ -2004,7 +2077,7 @@ func TestSessionInvalidate_DeleteNodeService(t *testing.T) {
 		ServiceID: "api",
 	}
 	if err := store.EnsureCheck(13, check); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	session := &structs.Session{
@@ -2039,7 +2112,7 @@ func TestKVSLock(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	session := &structs.Session{ID: generateUUID(), Node: "foo"}
 	if err := store.SessionCreate(4, session); err != nil {
@@ -2112,7 +2185,7 @@ func TestKVSUnlock(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	session := &structs.Session{ID: generateUUID(), Node: "foo"}
 	if err := store.SessionCreate(4, session); err != nil {
@@ -2170,7 +2243,7 @@ func TestSessionInvalidate_KeyUnlock(t *testing.T) {
 	defer store.Close()
 
 	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 	session := &structs.Session{
 		ID:        generateUUID(),
@@ -2198,7 +2271,7 @@ func TestSessionInvalidate_KeyUnlock(t *testing.T) {
 
 	// Delete the node
 	if err := store.DeleteNode(6, "foo"); err != nil {
-		t.Fatalf("err: %v")
+		t.Fatalf("err: %v", err)
 	}
 
 	// Key should be unlocked
@@ -2220,6 +2293,53 @@ func TestSessionInvalidate_KeyUnlock(t *testing.T) {
 	}
 }
 
+func TestSessionInvalidate_KeyDelete(t *testing.T) {
+	store, err := testStateStore()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer store.Close()
+
+	if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	session := &structs.Session{
+		ID:        generateUUID(),
+		Node:      "foo",
+		LockDelay: 50 * time.Millisecond,
+		Behavior:  structs.SessionKeysDelete,
+	}
+	if err := store.SessionCreate(4, session); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Lock a key with the session
+	d := &structs.DirEntry{
+		Key:     "/bar",
+		Flags:   42,
+		Value:   []byte("test"),
+		Session: session.ID,
+	}
+	ok, err := store.KVSLock(5, d)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !ok {
+		t.Fatalf("unexpected fail")
+	}
+
+	// Delete the node
+	if err := store.DeleteNode(6, "foo"); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Key should be deleted
+	_, d2, err := store.KVSGet("/bar")
+	if d2 != nil {
+		t.Fatalf("unexpected undeleted key")
+	}
+}
+
 func TestACLSet_Get(t *testing.T) {
 	store, err := testStateStore()
 	if err != nil {

consul/status_endpoint_test.go

@@ -2,7 +2,7 @@ package consul
 
 import (
 	"github.com/hashicorp/consul/testutil"
-	"github.com/ugorji/go/codec"
+	"github.com/hashicorp/go-msgpack/codec"
 	"net"
 	"net/rpc"
 	"os"

consul/structs/structs.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/acl"
-	"github.com/ugorji/go/codec"
+	"github.com/hashicorp/go-msgpack/codec"
 )
 
 var (
@@ -378,6 +378,19 @@ type IndexedKeyList struct {
 	QueryMeta
 }
 
+type SessionBehavior string
+
+const (
+	SessionKeysRelease SessionBehavior = "release"
+	SessionKeysDelete                  = "delete"
+)
+
+const (
+	SessionTTLMin        = 10 * time.Second
+	SessionTTLMax        = 3600 * time.Second
+	SessionTTLMultiplier = 2
+)
+
 // Session is used to represent an open session in the KV store.
 // This issued to associate node checks with acquired locks.
 type Session struct {
@@ -387,6 +400,8 @@ type Session struct {
 	Node        string
 	Checks      []string
 	LockDelay   time.Duration
+	Behavior    SessionBehavior // What to do when session is invalidated
+	TTL         string
 }
 type Sessions []*Session
 
@@ -531,3 +546,66 @@ func Encode(t MessageType, msg interface{}) ([]byte, error) {
 	err := codec.NewEncoder(&buf, msgpackHandle).Encode(msg)
 	return buf.Bytes(), err
 }
+
+// CompoundResponse is an interface for gathering multiple responses. It is
+// used in cross-datacenter RPC calls where more than 1 datacenter is
+// expected to reply.
+type CompoundResponse interface {
+	// Add adds a new response to the compound response
+	Add(interface{})
+
+	// New returns an empty response object which can be passed around by
+	// reference, and then passed to Add() later on.
+	New() interface{}
+}
+
+type KeyringOp string
+
+const (
+	KeyringList    KeyringOp = "list"
+	KeyringInstall           = "install"
+	KeyringUse               = "use"
+	KeyringRemove            = "remove"
+)
+
+// KeyringRequest encapsulates a request to modify an encryption keyring.
+// It can be used for install, remove, or use key type operations.
+type KeyringRequest struct {
+	Operation  KeyringOp
+	Key        string
+	Datacenter string
+	Forwarded  bool
+	QueryOptions
+}
+
+func (r *KeyringRequest) RequestDatacenter() string {
+	return r.Datacenter
+}
+
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	WAN        bool
+	Datacenter string
+	Messages   map[string]string
+	Keys       map[string]int
+	NumNodes   int
+	Error      string
+}
+
+// KeyringResponses holds multiple responses to keyring queries. Each
+// datacenter replies independently, and KeyringResponses is used as a
+// container for the set of all responses.
+type KeyringResponses struct {
+	Responses []*KeyringResponse
+	QueryMeta
+}
+
+func (r *KeyringResponses) Add(v interface{}) {
+	val := v.(*KeyringResponses)
+	r.Responses = append(r.Responses, val.Responses...)
+}
+
+func (r *KeyringResponses) New() interface{} {
+	return new(KeyringResponses)
+}

consul/structs/structs_test.go

@@ -1 +1,54 @@
 package structs
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	arg := &RegisterRequest{
+		Datacenter: "foo",
+		Node:       "bar",
+		Address:    "baz",
+		Service: &NodeService{
+			Service: "test",
+		},
+	}
+	buf, err := Encode(RegisterRequestType, arg)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	var out RegisterRequest
+	err = Decode(buf[1:], &out)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !reflect.DeepEqual(arg.Service, out.Service) {
+		t.Fatalf("bad: %#v %#v", arg.Service, out.Service)
+	}
+	if !reflect.DeepEqual(arg, &out) {
+		t.Fatalf("bad: %#v %#v", arg, out)
+	}
+}
+
+func TestStructs_Implements(t *testing.T) {
+	var (
+		_ RPCInfo          = &RegisterRequest{}
+		_ RPCInfo          = &DeregisterRequest{}
+		_ RPCInfo          = &DCSpecificRequest{}
+		_ RPCInfo          = &ServiceSpecificRequest{}
+		_ RPCInfo          = &NodeSpecificRequest{}
+		_ RPCInfo          = &ChecksInStateRequest{}
+		_ RPCInfo          = &KVSRequest{}
+		_ RPCInfo          = &KeyRequest{}
+		_ RPCInfo          = &KeyListRequest{}
+		_ RPCInfo          = &SessionRequest{}
+		_ RPCInfo          = &SessionSpecificRequest{}
+		_ RPCInfo          = &EventFireRequest{}
+		_ RPCInfo          = &ACLPolicyRequest{}
+		_ RPCInfo          = &KeyringRequest{}
+		_ CompoundResponse = &KeyringResponses{}
+	)
+}

consul/util_test.go

@@ -97,7 +97,7 @@ func TestIsConsulNode(t *testing.T) {
 	}
 	valid, dc := isConsulNode(m)
 	if !valid || dc != "east-aws" {
-		t.Fatalf("bad: %v %v %v", valid, dc)
+		t.Fatalf("bad: %v %v", valid, dc)
 	}
 }
 

demo/vagrant-cluster/Vagrantfile

@@ -3,11 +3,11 @@
 $script = <<SCRIPT
 
 echo Installing dependencies...
-sudo apt-get install -y unzip
+sudo apt-get install -y unzip curl
 
 echo Fetching Consul...
 cd /tmp/
-wget https://dl.bintray.com/mitchellh/consul/0.4.0_linux_amd64.zip -O consul.zip
+wget https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip -O consul.zip
 
 echo Installing Consul...
 unzip consul.zip

deps/v0-4-1.json

@@ -0,0 +1,97 @@
+{
+	"ImportPath": "github.com/hashicorp/consul",
+	"GoVersion": "go1.3.3",
+	"Deps": [
+		{
+			"ImportPath": "github.com/armon/circbuf",
+			"Rev": "f092b4f207b6e5cce0569056fba9e1a2735cb6cf"
+		},
+		{
+			"ImportPath": "github.com/armon/consul-api",
+			"Rev": "1b81c8e0c4cbf1d382310e4c0dc11221632e79d1"
+		},
+		{
+			"ImportPath": "github.com/armon/go-metrics",
+			"Rev": "2b75159ce5d3641fb35b5a159cff309ac3cf4177"
+		},
+		{
+			"ImportPath": "github.com/armon/go-radix",
+			"Rev": "b045fc0ad3587e8620fb42a0dea882cf8c08aef9"
+		},
+		{
+			"ImportPath": "github.com/armon/gomdb",
+			"Rev": "a8e036c4dabe7437014ecf9dbc03c6f6f0766ef8"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/go-checkpoint",
+			"Rev": "89ef2a697dd8cdb4623097d5bb9acdb19a470767"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/go-msgpack/codec",
+			"Rev": "71c2886f5a673a35f909803f38ece5810165097b"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/go-syslog",
+			"Rev": "ac3963b72ac367e48b1e68a831e62b93fb69091c"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/golang-lru",
+			"Rev": "253b2dc1ca8bae42c3b5b6e53dd2eab1a7551116"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/hcl",
+			"Rev": "e51eabcdf801f663738fa12f4340fbad13062738"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/logutils",
+			"Rev": "23b0af5510a2d1442103ef83ffcf53eb82f3debc"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/memberlist",
+			"Rev": "16d947e2d4b3f1fe508ee1d9b6ec34b8fd2e96d8"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/raft",
+			"Rev": "cc9710ab540985954a67c108f414aa3152f5916f"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/raft-mdb",
+			"Rev": "6f52d0ce62a34e3f5bd29aa4d7068030d700d94a"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/serf/serf",
+			"Comment": "v0.6.3-60-g0479bc1",
+			"Rev": "0479bc1b942fd84205587f7e73867ac78809966b"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/terraform/helper/multierror",
+			"Comment": "v0.3.1-25-g2d11732",
+			"Rev": "2d117326edb33b7155d1ec9d0ab9d3542ba1b230"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/yamux",
+			"Rev": "9feabe6854fadca1abec9cd3bd2a613fe9a34000"
+		},
+		{
+			"ImportPath": "github.com/inconshreveable/muxado",
+			"Rev": "f693c7e88ba316d1a0ae3e205e22a01aa3ec2848"
+		},
+		{
+			"ImportPath": "github.com/miekg/dns",
+			"Rev": "dc30c7cd4ed2fc8af73d49da4ee285404958b8bd"
+		},
+		{
+			"ImportPath": "github.com/mitchellh/cli",
+			"Rev": "e3c2e3d39391e9beb9660ccd6b4bd9a2f38dd8a0"
+		},
+		{
+			"ImportPath": "github.com/mitchellh/mapstructure",
+			"Rev": "740c764bc6149d3f1806231418adb9f52c11bcbf"
+		},
+		{
+			"ImportPath": "github.com/ryanuber/columnize",
+			"Comment": "v2.0.1-6-g44cb478",
+			"Rev": "44cb4788b2ec3c3d158dd3d1b50aba7d66f4b59a"
+		}
+	]
+}

scripts/build.sh

@@ -14,6 +14,7 @@ cd $DIR
 # Get the git commit
 GIT_COMMIT=$(git rev-parse HEAD)
 GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)
+GIT_DESCRIBE=$(git describe --tags)
 
 # If we're building on Windows, specify an extension
 EXTENSION=""
@@ -45,7 +46,7 @@ go get \
 # Build!
 echo "--> Building..."
 go build \
-    -ldflags "${CGO_LDFLAGS} -X main.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \
+    -ldflags "${CGO_LDFLAGS} -X main.GitCommit ${GIT_COMMIT}${GIT_DIRTY} -X main.GitDescribe ${GIT_DESCRIBE}" \
     -v \
     -o bin/consul${EXTENSION}
 cp bin/consul${EXTENSION} ${GOPATHSINGLE}/bin

terraform/aws/scripts/install.sh

@@ -10,7 +10,7 @@ sudo apt-get install -y unzip
 
 echo "Fetching Consul..."
 cd /tmp
-wget https://dl.bintray.com/mitchellh/consul/0.4.0_linux_amd64.zip -O consul.zip
+wget https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip -O consul.zip
 
 echo "Installing Consul..."
 unzip consul.zip >/dev/null

terraform/aws/scripts/server.sh

@@ -9,6 +9,6 @@ cat >/tmp/consul_flags << EOF
 export CONSUL_FLAGS="-server -bootstrap-expect=${SERVER_COUNT} -data-dir=/mnt/consul"
 EOF
 
-# Write it to the full sevice file
+# Write it to the full service file
 sudo mv /tmp/consul_flags /etc/service/consul
 chmod 0644 /etc/service/consul

tlsutil/config.go

@@ -0,0 +1,206 @@
+package tlsutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"time"
+)
+
+// Config used to create tls.Config
+type Config struct {
+	// VerifyIncoming is used to verify the authenticity of incoming connections.
+	// This means that TCP requests are forbidden, only allowing for TLS. TLS connections
+	// must match a provided certificate authority. This can be used to force client auth.
+	VerifyIncoming bool
+
+	// VerifyOutgoing is used to verify the authenticity of outgoing connections.
+	// This means that TLS requests are used, and TCP requests are not made. TLS connections
+	// must match a provided certificate authority. This is used to verify authenticity of
+	// server nodes.
+	VerifyOutgoing bool
+
+	// CAFile is a path to a certificate authority file. This is used with VerifyIncoming
+	// or VerifyOutgoing to verify the TLS connection.
+	CAFile string
+
+	// CertFile is used to provide a TLS certificate that is used for serving TLS connections.
+	// Must be provided to serve TLS connections.
+	CertFile string
+
+	// KeyFile is used to provide a TLS key that is used for serving TLS connections.
+	// Must be provided to serve TLS connections.
+	KeyFile string
+
+	// Node name is the name we use to advertise. Defaults to hostname.
+	NodeName string
+
+	// ServerName is used with the TLS certificate to ensure the name we
+	// provide matches the certificate
+	ServerName string
+}
+
+// AppendCA opens and parses the CA file and adds the certificates to
+// the provided CertPool.
+func (c *Config) AppendCA(pool *x509.CertPool) error {
+	if c.CAFile == "" {
+		return nil
+	}
+
+	// Read the file
+	data, err := ioutil.ReadFile(c.CAFile)
+	if err != nil {
+		return fmt.Errorf("Failed to read CA file: %v", err)
+	}
+
+	if !pool.AppendCertsFromPEM(data) {
+		return fmt.Errorf("Failed to parse any CA certificates")
+	}
+
+	return nil
+}
+
+// KeyPair is used to open and parse a certificate and key file
+func (c *Config) KeyPair() (*tls.Certificate, error) {
+	if c.CertFile == "" || c.KeyFile == "" {
+		return nil, nil
+	}
+	cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to load cert/key pair: %v", err)
+	}
+	return &cert, err
+}
+
+// OutgoingTLSConfig generates a TLS configuration for outgoing
+// requests. It will return a nil config if this configuration should
+// not use TLS for outgoing connections.
+func (c *Config) OutgoingTLSConfig() (*tls.Config, error) {
+	if !c.VerifyOutgoing {
+		return nil, nil
+	}
+	// Create the tlsConfig
+	tlsConfig := &tls.Config{
+		RootCAs:            x509.NewCertPool(),
+		InsecureSkipVerify: true,
+	}
+	if c.ServerName != "" {
+		tlsConfig.ServerName = c.ServerName
+		tlsConfig.InsecureSkipVerify = false
+	}
+
+	// Ensure we have a CA if VerifyOutgoing is set
+	if c.VerifyOutgoing && c.CAFile == "" {
+		return nil, fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!")
+	}
+
+	// Parse the CA cert if any
+	err := c.AppendCA(tlsConfig.RootCAs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add cert/key
+	cert, err := c.KeyPair()
+	if err != nil {
+		return nil, err
+	} else if cert != nil {
+		tlsConfig.Certificates = []tls.Certificate{*cert}
+	}
+
+	return tlsConfig, nil
+}
+
+// Wrap a net.Conn into a client tls connection, performing any
+// additional verification as needed.
+//
+// As of go 1.3, crypto/tls only supports either doing no certificate
+// verification, or doing full verification including of the peer's
+// DNS name. For consul, we want to validate that the certificate is
+// signed by a known CA, but because consul doesn't use DNS names for
+// node names, we don't verify the certificate DNS names. Since go 1.3
+// no longer supports this mode of operation, we have to do it
+// manually.
+func WrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
+	var err error
+	var tlsConn *tls.Conn
+
+	tlsConn = tls.Client(conn, tlsConfig)
+
+	// If crypto/tls is doing verification, there's no need to do
+	// our own.
+	if tlsConfig.InsecureSkipVerify == false {
+		return tlsConn, nil
+	}
+
+	if err = tlsConn.Handshake(); err != nil {
+		tlsConn.Close()
+		return nil, err
+	}
+
+	// The following is lightly-modified from the doFullHandshake
+	// method in crypto/tls's handshake_client.go.
+	opts := x509.VerifyOptions{
+		Roots:         tlsConfig.RootCAs,
+		CurrentTime:   time.Now(),
+		DNSName:       "",
+		Intermediates: x509.NewCertPool(),
+	}
+
+	certs := tlsConn.ConnectionState().PeerCertificates
+	for i, cert := range certs {
+		if i == 0 {
+			continue
+		}
+		opts.Intermediates.AddCert(cert)
+	}
+
+	_, err = certs[0].Verify(opts)
+	if err != nil {
+		tlsConn.Close()
+		return nil, err
+	}
+
+	return tlsConn, err
+}
+
+// IncomingTLSConfig generates a TLS configuration for incoming requests
+func (c *Config) IncomingTLSConfig() (*tls.Config, error) {
+	// Create the tlsConfig
+	tlsConfig := &tls.Config{
+		ServerName: c.ServerName,
+		ClientCAs:  x509.NewCertPool(),
+		ClientAuth: tls.NoClientCert,
+	}
+	if tlsConfig.ServerName == "" {
+		tlsConfig.ServerName = c.NodeName
+	}
+
+	// Parse the CA cert if any
+	err := c.AppendCA(tlsConfig.ClientCAs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add cert/key
+	cert, err := c.KeyPair()
+	if err != nil {
+		return nil, err
+	} else if cert != nil {
+		tlsConfig.Certificates = []tls.Certificate{*cert}
+	}
+
+	// Check if we require verification
+	if c.VerifyIncoming {
+		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+		if c.CAFile == "" {
+			return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!")
+		}
+		if cert == nil {
+			return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!")
+		}
+	}
+	return tlsConfig, nil
+}

tlsutil/config_test.go

@@ -1,4 +1,4 @@
-package consul
+package tlsutil
 
 import (
 	"crypto/tls"
@@ -204,7 +204,7 @@ func TestConfig_wrapTLS_OK(t *testing.T) {
 		t.Fatalf("OutgoingTLSConfig err: %v", err)
 	}
 
-	tlsClient, err := wrapTLSClient(client, clientConfig)
+	tlsClient, err := WrapTLSClient(client, clientConfig)
 	if err != nil {
 		t.Fatalf("wrapTLS err: %v", err)
 	} else {
@@ -237,7 +237,7 @@ func TestConfig_wrapTLS_BadCert(t *testing.T) {
 		t.Fatalf("OutgoingTLSConfig err: %v", err)
 	}
 
-	tlsClient, err := wrapTLSClient(client, clientTLSConfig)
+	tlsClient, err := WrapTLSClient(client, clientTLSConfig)
 	if err == nil {
 		t.Fatalf("wrapTLS no err")
 	}

ui/javascripts/app/controllers.js

@@ -161,23 +161,25 @@ App.KvShowController.reopen({
     },
 
     deleteFolder: function() {
-      this.set('isLoading', true);
 
+      this.set('isLoading', true);
       var controller = this;
       var dc = controller.get('dc').get('datacenter');
       var grandParent = controller.get('grandParentKey');
       var token = App.get('settings.token');
 
-      // Delete the folder
-      Ember.$.ajax({
-          url: (formatUrl("/v1/kv/" + controller.get('parentKey') + '?recurse', dc, token)),
-          type: 'DELETE'
-      }).then(function(response) {
-        controller.transitionToNearestParent(grandParent);
-      }).fail(function(response) {
-        // Render the error message on the form if the request failed
-        controller.set('errorMessage', 'Received error while processing: ' + response.statusText);
-      });
+      if (window.confirm("Are you sure you want to delete this folder?")) {
+        // Delete the folder
+        Ember.$.ajax({
+            url: (formatUrl("/v1/kv/" + controller.get('parentKey') + '?recurse', dc, token)),
+            type: 'DELETE'
+        }).then(function(response) {
+          controller.transitionToNearestParent(grandParent);
+        }).fail(function(response) {
+          // Render the error message on the form if the request failed
+          controller.set('errorMessage', 'Received error while processing: ' + response.statusText);
+        });
+      }
     }
   }
 });
@@ -460,7 +462,7 @@ App.AclsShowController = Ember.ObjectController.extend({
       }).then(function(response) {
         controller.transitionToRoute('acls.show', response.ID);
         controller.set('isLoading', false);
-        notify('Succesfully cloned token', 4000);
+        notify('Successfully cloned token', 4000);
       }).fail(function(response) {
         // Render the error message on the form if the request failed
         controller.set('errorMessage', 'Received error while processing: ' + response.statusText);

ui/javascripts/libs/classie.js

@@ -22,7 +22,7 @@ function classReg( className ) {
 }
 
 // classList support for class management
-// altho to be fair, the api sucks because it won't accept multiple classes at once
+// although to be fair, the api sucks because it won't accept multiple classes at once
 var hasClass, addClass, removeClass;
 
 if ( 'classList' in document.documentElement ) {

version.go

@@ -2,6 +2,7 @@ package main
 
 // The git commit that was compiled. This will be filled in by the compiler.
 var GitCommit string
+var GitDescribe string
 
 // The main version number that is being run at the moment.
 const Version = "0.4.1"
@@ -9,4 +10,4 @@ const Version = "0.4.1"
 // A pre-release marker for the version. If this is "" (empty string)
 // then it means that it is a final release. Otherwise, this is a pre-release
 // such as "dev" (in development), "beta", "rc1", etc.
-const VersionPrerelease = "rc"
+const VersionPrerelease = ""

watch/watch.go

@@ -107,7 +107,7 @@ func assignValue(params map[string]interface{}, name string, out *string) error
 	if raw, ok := params[name]; ok {
 		val, ok := raw.(string)
 		if !ok {
-			return fmt.Errorf("Expecting %s to be a string")
+			return fmt.Errorf("Expecting %s to be a string", name)
 		}
 		*out = val
 		delete(params, name)
@@ -120,7 +120,7 @@ func assignValueBool(params map[string]interface{}, name string, out *bool) erro
 	if raw, ok := params[name]; ok {
 		val, ok := raw.(bool)
 		if !ok {
-			return fmt.Errorf("Expecting %s to be a boolean")
+			return fmt.Errorf("Expecting %s to be a boolean", name)
 		}
 		*out = val
 		delete(params, name)

website/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: git://github.com/hashicorp/middleman-hashicorp.git
-  revision: fe7d5bb4b04c408857dbe94345341cafcbc02de4
+  revision: b82c2c2fdc244cd0bd529ff27cfab24e43f07708
   specs:
     middleman-hashicorp (0.1.0)
       bootstrap-sass (~> 3.2)
@@ -11,6 +11,8 @@ GIT
       middleman-minify-html (~> 3.4)
       middleman-syntax (~> 2.0)
       rack-contrib (~> 1.1)
+      rack-rewrite (~> 1.5)
+      rack-ssl-enforcer (~> 0.2)
       redcarpet (~> 3.1)
       therubyracer (~> 0.12)
       thin (~> 1.6)
@@ -29,7 +31,7 @@ GEM
     builder (3.2.2)
     celluloid (0.16.0)
       timers (~> 4.0.0)
-    chunky_png (1.3.1)
+    chunky_png (1.3.3)
     coffee-script (2.3.0)
       coffee-script-source
       execjs
@@ -53,8 +55,8 @@ GEM
       http_parser.rb (~> 0.6.0)
     erubis (2.7.0)
     eventmachine (1.0.3)
-    execjs (2.2.1)
-    ffi (1.9.5)
+    execjs (2.2.2)
+    ffi (1.9.6)
     haml (4.0.5)
       tilt
     hike (1.2.3)
@@ -65,7 +67,7 @@ GEM
     http_parser.rb (0.6.0)
     i18n (0.6.11)
     json (1.8.1)
-    kramdown (1.4.2)
+    kramdown (1.5.0)
     less (2.6.0)
       commonjs (~> 0.2.7)
     libv8 (3.16.14.7)
@@ -113,26 +115,28 @@ GEM
       rouge (~> 1.0)
     minitest (5.4.2)
     multi_json (1.10.1)
-    padrino-helpers (0.12.3)
+    padrino-helpers (0.12.4)
       i18n (~> 0.6, >= 0.6.7)
-      padrino-support (= 0.12.3)
+      padrino-support (= 0.12.4)
       tilt (~> 1.4.1)
-    padrino-support (0.12.3)
+    padrino-support (0.12.4)
       activesupport (>= 3.1)
     rack (1.5.2)
     rack-contrib (1.1.0)
       rack (>= 0.9.1)
     rack-livereload (0.3.15)
       rack
+    rack-rewrite (1.5.0)
+    rack-ssl-enforcer (0.2.8)
     rack-test (0.6.2)
       rack (>= 1.0)
     rb-fsevent (0.9.4)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
-    redcarpet (3.1.2)
+    redcarpet (3.2.0)
     ref (1.0.5)
     rouge (1.7.2)
-    sass (3.4.5)
+    sass (3.4.6)
     sprockets (2.12.2)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -157,7 +161,7 @@ GEM
       hitimes
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    uber (0.0.8)
+    uber (0.0.10)
     uglifier (2.5.3)
       execjs (>= 0.3.0)
       json (>= 1.8.0)

website/config.rb

@@ -2,8 +2,10 @@
 # Configure Middleman
 #-------------------------------------------------------------------------
 
+set :base_url, "https://www.consul.io/"
+
 activate :hashicorp do |h|
-  h.version      = '0.4.0'
+  h.version      = '0.4.1'
   h.bintray_repo = 'mitchellh/consul'
   h.bintray_user = 'mitchellh'
   h.bintray_key  = ENV['BINTRAY_API_KEY']

website/config.ru

@@ -4,6 +4,11 @@ require "rack/contrib/response_headers"
 require "rack/contrib/static_cache"
 require "rack/contrib/try_static"
 
+require "rake/rewrite"
+use Rack::Rewrite do
+  r301 "/downloads_web_ui.html", "/downloads.html"
+end
+
 # Properly compress the output if the client can handle it.
 use Rack::Deflater
 

website/source/404.html.erb

@@ -1 +1,5 @@
+---
+noindex: true
+---
+
 <h2>Page Not Found</h2>

website/source/assets/fonts/league_gothic-webfont.svg

@@ -1,230 +0,0 @@
-<?xml version="1.0" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" >
-<svg xmlns="http://www.w3.org/2000/svg">
-<metadata></metadata>
-<defs>
-<font id="LeagueGothicRegular" horiz-adv-x="724" >
-<font-face units-per-em="2048" ascent="1505" descent="-543" />
-<missing-glyph horiz-adv-x="315" />
-<glyph horiz-adv-x="0" />
-<glyph horiz-adv-x="2048" />
-<glyph unicode="&#xd;" horiz-adv-x="682" />
-<glyph unicode=" "  horiz-adv-x="315" />
-<glyph unicode="&#x09;" horiz-adv-x="315" />
-<glyph unicode="&#xa0;" horiz-adv-x="315" />
-<glyph unicode="!" horiz-adv-x="387" d="M74 1505h239l-55 -1099h-129zM86 0v227h215v-227h-215z" />
-<glyph unicode="&#x22;" horiz-adv-x="329" d="M57 1505h215l-30 -551h-154z" />
-<glyph unicode="#" horiz-adv-x="1232" d="M49 438l27 195h198l37 258h-196l26 194h197l57 420h197l-57 -420h260l57 420h197l-58 -420h193l-27 -194h-192l-37 -258h190l-26 -195h-191l-59 -438h-197l60 438h-261l-59 -438h-197l60 438h-199zM471 633h260l37 258h-260z" />
-<glyph unicode="$" horiz-adv-x="692" d="M37 358l192 13q12 -186 129 -187q88 0 93 185q0 74 -61 175q-21 36 -34 53l-40 55q-28 38 -65.5 90t-70.5 101.5t-70.5 141.5t-37.5 170q4 293 215 342v131h123v-125q201 -23 235 -282l-192 -25q-14 129 -93 125q-80 -2 -84 -162q0 -102 94 -227l41 -59q30 -42 37 -52 t33 -48l37 -52q41 -57 68 -109l26 -55q43 -94 43 -186q-4 -338 -245 -369v-217h-123v221q-236 41 -250 352z" />
-<glyph unicode="%" horiz-adv-x="1001" d="M55 911v437q0 110 82 156q33 18 90.5 18t97.5 -44t44 -87l4 -43v-437q0 -107 -81 -157q-32 -19 -77 -19q-129 0 -156 135zM158 0l553 1505h131l-547 -1505h-137zM178 911q-4 -55 37 -55q16 0 25.5 14.5t9.5 26.5v451q2 55 -35 55q-18 0 -27.5 -13.5t-9.5 -27.5v-451z M631 158v436q0 108 81 156q33 20 79 20q125 0 153 -135l4 -41v-436q0 -110 -80 -156q-32 -18 -90.5 -18t-98.5 43t-44 88zM754 158q-4 -57 37 -58q37 0 34 58v436q2 55 -34 55q-18 0 -27.5 -13t-9.5 -28v-450z" />
-<glyph unicode="&#x26;" horiz-adv-x="854" d="M49 304q0 126 44 225.5t126 222.5q-106 225 -106 442v18q0 94 47 180q70 130 223 130q203 0 252 -215q14 -61 12 -113q0 -162 -205 -434q76 -174 148 -285q33 96 47 211l176 -33q-16 -213 -92 -358q55 -63 92 -76v-235q-23 0 -86 37.5t-123 101.5q-123 -139 -252 -139 t-216 97t-87 223zM263 325.5q1 -65.5 28.5 -107.5t78.5 -42t117 86q-88 139 -174 295q-18 -30 -34.5 -98t-15.5 -133.5zM305 1194q0 -111 55 -246q101 156 101 252q-2 2 0 15.5t-2 36t-11 42.5q-19 52 -61.5 52t-62 -38t-19.5 -75v-39z" />
-<glyph unicode="'" horiz-adv-x="309" d="M45 1012l72 266h-72v227h215v-227l-113 -266h-102z" />
-<glyph unicode="(" horiz-adv-x="561" d="M66 645q0 143 29.5 292.5t73.5 261.5q92 235 159 343l30 47l162 -84q-38 -53 -86.5 -148t-82.5 -189.5t-61.5 -238t-27.5 -284.5t26.5 -282.5t64.5 -240.5q80 -207 141 -296l26 -39l-162 -84q-41 61 -96 173t-94 217.5t-70.5 257t-31.5 294.5z" />
-<glyph unicode=")" horiz-adv-x="561" d="M41 -213q36 50 85.5 147t83.5 190t61.5 236.5t27.5 284.5t-26.5 282.5t-64.5 240.5q-78 205 -140 298l-27 39l162 84q41 -61 96 -173.5t94 -217t71 -257.5t32 -296t-30 -292.5t-74 -260.5q-92 -233 -159 -342l-30 -47z" />
-<glyph unicode="*" horiz-adv-x="677" d="M74 1251l43 148l164 -70l-19 176h154l-19 -176l164 70l43 -148l-172 -34l115 -138l-131 -80l-78 152l-76 -152l-131 80l115 138z" />
-<glyph unicode="+" horiz-adv-x="1060" d="M74 649v172h370v346h172v-346h371v-172h-371v-346h-172v346h-370z" />
-<glyph unicode="," horiz-adv-x="309" d="M45 0v227h215v-227l-113 -266h-102l72 266h-72z" />
-<glyph unicode="-" horiz-adv-x="444" d="M74 455v194h297v-194h-297z" />
-<glyph unicode="." horiz-adv-x="321" d="M53 0v227h215v-227h-215z" />
-<glyph unicode="/" horiz-adv-x="720" d="M8 -147l543 1652h162l-537 -1652h-168z" />
-<glyph unicode="0" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5l2 26v887q0 46 -19.5 78.5 t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="1" horiz-adv-x="475" d="M25 1180v141q129 25 205 130q16 21 30 54h133v-1505h-221v1180h-147z" />
-<glyph unicode="2" horiz-adv-x="731" d="M55 0v219l39 62q25 39 88.5 152.5t112.5 220t91 241.5t44 238q0 184 -73.5 184t-73.5 -184v-105h-222v105q0 389 295 389t295 -375q0 -336 -346 -928h350v-219h-600z" />
-<glyph unicode="3" horiz-adv-x="686" d="M45 1071q0 249 63 343q29 42 84.5 75t134.5 33t136 -31t84.5 -71t44.5 -92q22 -71 22 -130q0 -291 -108 -399q127 -100 127 -414q0 -68 -19.5 -145.5t-47 -128t-85 -89t-136.5 -38.5t-135 31.5t-86 75.5t-48 113q-23 91 -23 230h217q2 -150 17.5 -203t59.5 -53t56.5 50.5 t12.5 104.5t1 102t0 63q-6 82 -14 95l-18 33q-12 22 -29 29q-55 22 -108 25h-19v184q133 7 156 73q12 34 12 91v105q0 146 -29 177q-16 17 -40 17q-41 0 -52.5 -49t-13.5 -207h-217z" />
-<glyph unicode="4" horiz-adv-x="684" d="M25 328v194l323 983h221v-983h103v-194h-103v-328h-202v328h-342zM213 522h154v516h-13z" />
-<glyph unicode="5" horiz-adv-x="704" d="M74 438h221v-59q0 -115 14.5 -159t52 -44t53 45t15.5 156v336q0 111 -70 110q-33 0 -59.5 -40t-26.5 -70h-186v792h535v-219h-344v-313q74 55 127 51q78 0 133 -40t77 -100q35 -98 35 -171v-336q0 -393 -289 -393q-78 0 -133 29.5t-84.5 71.5t-46.5 109q-24 98 -24 244z " />
-<glyph unicode="6" horiz-adv-x="700" d="M66 309v856q0 356 288.5 356.5t288.5 -356.5v-94h-221q0 162 -11.5 210t-53.5 48t-56 -37t-14 -127v-268q59 37 124.5 37t119 -36t75.5 -93q37 -92 37 -189v-307q0 -90 -42 -187q-26 -61 -89 -99.5t-157.5 -38.5t-158 38.5t-88.5 99.5q-42 98 -42 187zM287 244 q0 -20 17.5 -44t49 -24t50 24.5t18.5 43.5v450q0 18 -18.5 43t-49 25t-48 -20.5t-19.5 -41.5v-456z" />
-<glyph unicode="7" horiz-adv-x="589" d="M8 1286v219h557v-221l-221 -1284h-229l225 1286h-332z" />
-<glyph unicode="8" horiz-adv-x="696" d="M53 322v176q0 188 115 297q-102 102 -102 276v127q0 213 147 293q57 31 135 31t135.5 -31t84 -71t42.5 -93q21 -66 21 -129v-127q0 -174 -103 -276q115 -109 115 -297v-176q0 -222 -153 -306q-60 -32 -142 -32t-141.5 32.5t-88 73.5t-44.5 96q-21 69 -21 136zM269 422 q1 -139 16.5 -187.5t57.5 -48.5t59.5 30t21.5 71t4 158t-13.5 174t-66.5 57t-66.5 -57.5t-12.5 -196.5zM284 1116q-1 -123 11 -173t53 -50t53.5 50t12.5 170t-12.5 167t-51.5 47t-52 -44t-14 -167z" />
-<glyph unicode="9" horiz-adv-x="700" d="M57 340v94h222q0 -162 11 -210t53 -48t56.5 37t14.5 127v283q-59 -37 -125 -37t-119 35.5t-76 92.5q-37 96 -37 189v293q0 87 43 188q25 60 88.5 99t157.5 39t157.5 -39t88.5 -99q43 -101 43 -188v-856q0 -356 -289 -356t-289 356zM279 825q0 -18 18 -42.5t49 -24.5 t48.5 20.5t19.5 40.5v443q0 20 -17.5 43.5t-49.5 23.5t-50 -24.5t-18 -42.5v-437z" />
-<glyph unicode=":" horiz-adv-x="362" d="M74 0v227h215v-227h-215zM74 893v227h215v-227h-215z" />
-<glyph unicode=";" horiz-adv-x="362" d="M74 0v227h215v-227l-113 -266h-102l71 266h-71zM74 893v227h215v-227h-215z" />
-<glyph unicode="&#x3c;" horiz-adv-x="1058" d="M74 649v160l911 475v-199l-698 -356l698 -356v-199z" />
-<glyph unicode="=" horiz-adv-x="1058" d="M74 477v172h911v-172h-911zM74 864v172h911v-172h-911z" />
-<glyph unicode="&#x3e;" horiz-adv-x="1058" d="M74 174v199l698 356l-698 356v199l911 -475v-160z" />
-<glyph unicode="?" horiz-adv-x="645" d="M25 1260q24 67 78 131q105 128 235 122q82 -2 138 -33.5t82 -81.5q46 -88 46 -170.5t-80 -219.5l-57 -96q-18 -32 -42 -106.5t-24 -143.5v-256h-190v256q0 102 24.5 195t48 140t65.5 118t50 105t-9 67.5t-60 34.5t-78 -48t-49 -98zM199 0h215v227h-215v-227z" />
-<glyph unicode="@" horiz-adv-x="872" d="M66 303v889q0 97 73 200q39 56 117 93t184.5 37t184 -37t116.5 -93q74 -105 74 -200v-793h-164l-20 56q-14 -28 -46 -48t-67 -20q-145 0 -145 172v485q0 170 145 170q71 0 113 -67v45q0 51 -45 104.5t-145.5 53.5t-145.5 -53.5t-45 -104.5v-889q0 -53 44 -103t153.5 -50 t160.5 63l152 -86q-109 -143 -320 -143q-106 0 -184 35.5t-117 90.5q-73 102 -73 193zM535 573q0 -53 48 -53t48 53v455q0 53 -48 53t-48 -53v-455z" />
-<glyph unicode="A" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM307 541h152l-64 475l-6 39h-12z" />
-<glyph unicode="B" horiz-adv-x="745" d="M82 0v1505h194q205 0 304.5 -91t99.5 -308q0 -106 -29.5 -175t-107.5 -136q14 -5 47 -38.5t54 -71.5q52 -97 52 -259q0 -414 -342 -426h-272zM303 219q74 0 109 31q55 56 55 211t-63 195q-42 26 -93 26h-8v-463zM303 885q87 0 119 39q45 55 45 138t-14.5 124t-30.5 60.5 t-45 28.5q-35 11 -74 11v-401z" />
-<glyph unicode="C" horiz-adv-x="708" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-207h-206v207q-2 0 0 11.5t-3.5 27.5t-12.5 33q-17 39 -68 39q-70 -10 -78 -111v-887q0 -43 21.5 -76.5t59.5 -33.5t59.5 27.5t21.5 56.5v233h206v-207q0 -42 -17 -106t-45 -107 t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175z" />
-<glyph unicode="D" horiz-adv-x="761" d="M82 0v1505h174q270 0 346 -113q31 -46 50.5 -95.5t28.5 -139.5t12 -177t3 -228.5t-3 -228.5t-12 -176t-28.5 -138t-50.5 -95t-80 -68q-106 -46 -266 -46h-174zM303 221q117 0 140.5 78t23.5 399v111q0 322 -23.5 398.5t-140.5 76.5v-1063z" />
-<glyph unicode="E" horiz-adv-x="628" d="M82 0v1505h506v-227h-285v-395h205v-242h-205v-414h285v-227h-506z" />
-<glyph unicode="F" horiz-adv-x="616" d="M82 0v1505h526v-227h-305v-395h205v-228h-205v-655h-221z" />
-<glyph unicode="G" horiz-adv-x="737" d="M67 271.5q0 26.5 1 37.5v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-231h-221v231q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-905q0 -46 19.5 -78.5t54 -32.5t53 28t18.5 54l2 29v272h-88v187h309v-750h-131l-26 72 q-70 -88 -172 -88q-203 0 -250 213q-11 48 -11 74.5z" />
-<glyph unicode="H" horiz-adv-x="778" d="M82 0v1505h221v-622h172v622h221v-1505h-221v655h-172v-655h-221z" />
-<glyph unicode="I" horiz-adv-x="385" d="M82 0v1505h221v-1505h-221z" />
-<glyph unicode="J" horiz-adv-x="423" d="M12 -14v217q4 0 12.5 -1t29 2t35.5 12t28.5 34.5t13.5 62.5v1192h221v-1226q0 -137 -74 -216q-74 -78 -223 -78h-4q-19 0 -39 1z" />
-<glyph unicode="K" horiz-adv-x="768" d="M82 0v1505h221v-526h8l195 526h215l-203 -495l230 -1010h-216l-153 655l-6 31h-6l-64 -154v-532h-221z" />
-<glyph unicode="L" horiz-adv-x="604" d="M82 0v1505h221v-1300h293v-205h-514z" />
-<glyph unicode="M" horiz-adv-x="991" d="M82 0v1505h270l131 -688l11 -80h4l10 80l131 688h270v-1505h-204v1010h-13l-149 -1010h-94l-142 946l-8 64h-12v-1010h-205z" />
-<glyph unicode="N" horiz-adv-x="808" d="M82 0v1505h197l215 -784l18 -70h12v854h203v-1505h-197l-215 784l-18 70h-12v-854h-203z" />
-<glyph unicode="O" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5l2 26v887q0 46 -19.5 78.5 t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="P" horiz-adv-x="720" d="M82 0v1505h221q166 0 277.5 -105.5t111.5 -345t-111.5 -346t-277.5 -106.5v-602h-221zM303 827q102 0 134 45.5t32 175.5t-33 181t-133 51v-453z" />
-<glyph unicode="Q" horiz-adv-x="729" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -94 -45 -182q33 -43 88 -53v-189q-160 0 -227 117q-55 -18 -125 -18t-130 33.5t-88 81.5q-55 94 -60 175zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5l2 26v887 q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="R" horiz-adv-x="739" d="M82 0v1505h221q377 0 377 -434q0 -258 -123 -342l141 -729h-221l-115 635h-59v-635h-221zM303 840q117 0 149 98q15 49 15 125t-15.5 125t-45.5 68q-44 30 -103 30v-446z" />
-<glyph unicode="S" horiz-adv-x="702" d="M37 422l217 20q0 -256 104 -256q90 0 91 166q0 59 -32 117.5t-45 79.5l-54 79q-40 58 -77 113t-73.5 117t-68 148.5t-31.5 162.5q0 139 71.5 245t216.5 108h10q88 0 152 -36t94 -100q54 -120 54 -264l-217 -20q0 217 -89 217q-75 -2 -75 -146q0 -59 23 -105 q32 -66 58 -104l197 -296q31 -49 67 -139.5t36 -166.5q0 -378 -306 -378h-2q-229 0 -290 188q-31 99 -31 250z" />
-<glyph unicode="T" horiz-adv-x="647" d="M4 1278v227h639v-227h-209v-1278h-221v1278h-209z" />
-<glyph unicode="U" horiz-adv-x="749" d="M80 309v1196h221v-1196q0 -46 19.5 -78t54.5 -32t53 27.5t18 56.5l3 26v1196h221v-1196q0 -42 -17.5 -106t-45 -107t-88 -77.5t-144.5 -34.5t-144.5 33.5t-88.5 81.5q-55 97 -60 175z" />
-<glyph unicode="V" horiz-adv-x="716" d="M18 1505h215l111 -827l8 -64h13l118 891h215l-229 -1505h-221z" />
-<glyph unicode="W" horiz-adv-x="1036" d="M25 1505h204l88 -782l5 -49h16l100 831h160l100 -831h17l92 831h205l-203 -1505h-172l-115 801h-8l-115 -801h-172z" />
-<glyph unicode="X" horiz-adv-x="737" d="M16 0l244 791l-240 714h218l120 -381l7 -18h8l127 399h217l-240 -714l244 -791h-217l-127 449l-4 18h-8l-132 -467h-217z" />
-<glyph unicode="Y" horiz-adv-x="700" d="M14 1505h217l111 -481l6 -14h4l6 14l111 481h217l-225 -864v-641h-221v641z" />
-<glyph unicode="Z" horiz-adv-x="626" d="M20 0v238l347 1048h-297v219h536v-219l-352 -1067h352v-219h-586z" />
-<glyph unicode="[" horiz-adv-x="538" d="M82 -213v1718h399v-196h-202v-1325h202v-197h-399z" />
-<glyph unicode="\" horiz-adv-x="792" d="M8 1692h162l614 -1872h-168z" />
-<glyph unicode="]" horiz-adv-x="538" d="M57 -16h203v1325h-203v196h400v-1718h-400v197z" />
-<glyph unicode="^" horiz-adv-x="1101" d="M53 809l381 696h234l381 -696h-199l-299 543l-299 -543h-199z" />
-<glyph unicode="_" horiz-adv-x="1210" d="M74 -154h1063v-172h-1063v172z" />
-<glyph unicode="`" horiz-adv-x="1024" d="M293 1489h215l106 -184h-159z" />
-<glyph unicode="a" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM252 291 q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250z" />
-<glyph unicode="b" horiz-adv-x="686" d="M82 0v1505h207v-458q88 90 165 90t117.5 -69t40.5 -150v-715q0 -82 -41 -150.5t-118 -68.5q-33 0 -74 22.5t-66 44.5l-24 23v-74h-207zM289 246q0 -29 19.5 -48.5t42 -19.5t39 19.5t16.5 48.5v628q0 29 -16.5 48.5t-39 19.5t-42 -21.5t-19.5 -46.5v-628z" />
-<glyph unicode="c" horiz-adv-x="645" d="M66 315v490q0 332 264 332q137 0 201.5 -71t64.5 -251v-88h-207v135q0 51 -12 70.5t-47 19.5q-58 0 -58 -90v-604q0 -90 58 -90q35 0 47 19.5t12 70.5v156h207v-109q0 -180 -64.5 -250.5t-201.5 -70.5q-264 0 -264 331z" />
-<glyph unicode="d" horiz-adv-x="686" d="M74 203v715q0 82 41 150.5t118 68.5q33 0 74 -22.5t66 -45.5l24 -22v458h207v-1505h-207v74q-88 -90 -165 -90t-117.5 68.5t-40.5 150.5zM281 246q0 -29 16 -48.5t38.5 -19.5t42 19.5t19.5 48.5v628q0 25 -19.5 46.5t-42 21.5t-38.5 -19.5t-16 -48.5v-628z" />
-<glyph unicode="e" horiz-adv-x="659" d="M66 279v563q0 36 16 94.5t42 97.5t81 71t129 32q199 0 252 -197q14 -51 14 -92v-326h-342v-256q0 -59 39 -88q16 -12 37 -12q70 10 74 113v122h192v-129q0 -37 -16.5 -93t-41 -95t-79.5 -69.5t-130 -30.5t-130.5 30.5t-80.5 73.5q-49 87 -54 160zM258 684h150v158 q0 48 -19.5 81t-53.5 33t-53.5 -28.5t-21.5 -57.5l-2 -28v-158z" />
-<glyph unicode="f" horiz-adv-x="475" d="M20 934v186h105v31q0 190 51 270q23 35 71 63.5t115 28.5l97 -14v-178q-27 8 -62 8q-65 0 -65 -175v-5v-29h104v-186h-104v-934h-207v934h-105z" />
-<glyph unicode="g" horiz-adv-x="700" d="M12 -184q0 94 162 170q-125 35 -125 149q0 45 40 93t89 75q-51 35 -80.5 95.5t-34.5 105.5l-4 43v305q0 35 16.5 91t41 94t79 69t126.5 31q135 0 206 -103q102 102 170 103v-185q-72 0 -120 -24l10 -70v-317q0 -37 -17.5 -90.5t-42 -90t-79 -66.5t-104.5 -30t-62 2 q-29 -25 -29 -46t11 -33.5t42 -20.5t45.5 -10t65.5 -10.5t95 -21.5t89 -41q96 -60 96 -205t-103 -212q-100 -65 -250 -65h-9q-156 2 -240 50t-84 165zM213 -150q0 -77 132 -77h3q59 0 108.5 19t49.5 54t-20.5 52.5t-90.5 29.5l-106 21q-76 -43 -76 -99zM262 509 q0 -17 15.5 -45t44.5 -28q63 6 63 101v307q-2 0 0 10q1 3 1 7q0 8 -3 19q-4 15 -9 30q-11 36 -46 36t-50.5 -25.5t-15.5 -52.5v-359z" />
-<glyph unicode="h" horiz-adv-x="690" d="M82 0v1505h207v-479l32 32q79 79 145.5 79t106 -69t39.5 -150v-918h-206v887q-1 49 -50 49q-41 0 -67 -53v-883h-207z" />
-<glyph unicode="i" horiz-adv-x="370" d="M82 0v1120h207v-1120h-207zM82 1298v207h207v-207h-207z" />
-<glyph unicode="j" horiz-adv-x="364" d="M-45 -182q29 -8 57 -8q64 0 64 142v1168h207v-1149q0 -186 -51 -266q-23 -35 -71 -62.5t-115 -27.5t-91 12v191zM76 1298v207h207v-207h-207z" />
-<glyph unicode="k" horiz-adv-x="641" d="M82 0v1505h207v-714h10l113 329h186l-149 -364l188 -756h-199l-102 453l-4 16h-10l-33 -82v-387h-207z" />
-<glyph unicode="l" horiz-adv-x="370" d="M82 0v1505h207v-1505h-207z" />
-<glyph unicode="m" horiz-adv-x="1021" d="M82 0v1120h207v-94q2 0 33 30q80 81 139 81q100 0 139 -125q125 125 194.5 125t109.5 -69t40 -150v-918h-194v887q-1 49 -56 49q-41 0 -78 -53v-883h-194v887q0 49 -55 49q-41 0 -78 -53v-883h-207z" />
-<glyph unicode="n" horiz-adv-x="690" d="M82 0v1120h207v-94l32 32q79 79 145.5 79t106 -69t39.5 -150v-918h-206v887q-1 49 -50 49q-41 0 -67 -53v-883h-207z" />
-<glyph unicode="o" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM257 259q0 -17 9 -44q18 -49 62 -49q70 10 71 113v563l1 19q0 19 -10 45q-18 50 -62 50 q-68 -10 -70 -114v-563q1 -1 1 -4z" />
-<glyph unicode="p" horiz-adv-x="686" d="M82 -385v1505h207v-73q88 90 165 90t117.5 -69t40.5 -150v-715q0 -82 -41 -150.5t-118 -68.5q-33 0 -74 22.5t-66 44.5l-24 23v-459h-207zM289 246q0 -25 19.5 -46.5t42 -21.5t39 19.5t16.5 48.5v628q0 29 -16.5 48.5t-39 19.5t-42 -19.5t-19.5 -48.5v-628z" />
-<glyph unicode="q" horiz-adv-x="686" d="M74 203v715q0 82 41 150.5t118 68.5q33 0 74 -22.5t66 -45.5l24 -22v73h207v-1505h-207v459q-88 -90 -165 -90t-117.5 68.5t-40.5 150.5zM281 246q0 -29 16 -48.5t38.5 -19.5t42 21.5t19.5 46.5v628q0 29 -19.5 48.5t-42 19.5t-38.5 -19.5t-16 -48.5v-628z" />
-<glyph unicode="r" horiz-adv-x="503" d="M82 0v1120h207v-125q8 41 58.5 91.5t148.5 50.5v-230q-34 11 -77 11t-86.5 -39t-43.5 -101v-778h-207z" />
-<glyph unicode="s" horiz-adv-x="630" d="M37 326h192q0 -170 97 -170q71 0 71 131q0 78 -129 202q-68 66 -98.5 99t-64 101.5t-33.5 134t12 114.5t39 95q59 100 201 104h11q161 0 211 -105q42 -86 42 -198h-193q0 131 -67 131q-63 -2 -64 -131q0 -33 23.5 -73t45 -62.5t66.5 -65.5q190 -182 191 -342 q0 -123 -64.5 -215t-199.5 -92q-197 0 -260 170q-29 76 -29 172z" />
-<glyph unicode="t" horiz-adv-x="501" d="M20 934v186h105v277h207v-277h141v-186h-141v-557q0 -184 65 -184l76 8v-203q-45 -14 -112 -14t-114.5 28.5t-70 64.5t-34.5 96q-17 79 -17 187v574h-105z" />
-<glyph unicode="u" horiz-adv-x="690" d="M78 203v917h207v-887q0 -49 49 -49q41 0 67 54v882h207v-1120h-207v94l-31 -32q-78 -78 -145.5 -78t-107 68.5t-39.5 150.5z" />
-<glyph unicode="v" horiz-adv-x="602" d="M16 1120h201l68 -649l8 -72h16l76 721h201l-183 -1120h-204z" />
-<glyph unicode="w" horiz-adv-x="905" d="M20 1120h189l65 -585l9 -64h12l96 649h123l86 -585l10 -64h13l73 649h189l-166 -1120h-172l-80 535l-10 63h-8l-91 -598h-172z" />
-<glyph unicode="x" horiz-adv-x="618" d="M16 0l193 578l-176 542h194l74 -262l6 -31h4l6 31l74 262h195l-176 -542l192 -578h-201l-84 283l-6 30h-4l-6 -30l-84 -283h-201z" />
-<glyph unicode="y" horiz-adv-x="634" d="M25 1120h202l82 -688l4 -57h9l4 57l82 688h202l-198 -1204q-16 -127 -94 -222t-193 -95l-92 4v184q16 -4 49 -4q61 6 97 61.5t36 122.5z" />
-<glyph unicode="z" horiz-adv-x="532" d="M12 0v168l285 764h-240v188h459v-168l-285 -764h285v-188h-504z" />
-<glyph unicode="{" horiz-adv-x="688" d="M61 453v163q72 0 102 49.5t30 90.5v397q0 223 96 298t342 71v-172q-135 2 -188.5 -38t-53.5 -159v-397q0 -143 -127 -221q127 -82 127 -222v-397q0 -119 53.5 -159t188.5 -38v-172q-246 -4 -342 71t-96 298v397q0 57 -41 97.5t-91 42.5z" />
-<glyph unicode="|" horiz-adv-x="356" d="M82 -512v2204h192v-2204h-192z" />
-<glyph unicode="}" horiz-adv-x="688" d="M57 -281q135 -2 188.5 38t53.5 159v397q0 139 127 222q-127 78 -127 221v397q0 119 -53 159t-189 38v172q246 4 342.5 -71t96.5 -298v-397q0 -63 41 -101.5t90 -38.5v-163q-72 -4 -101.5 -52.5t-29.5 -87.5v-397q0 -223 -96.5 -298t-342.5 -71v172z" />
-<glyph unicode="~" horiz-adv-x="1280" d="M113 1352q35 106 115 200q34 41 94.5 74t121 33t116.5 -18.5t82 -33t83 -51.5q106 -72 174 -71q109 0 178 153l13 29l135 -57q-63 -189 -206 -276q-56 -34 -120 -34q-121 0 -272 101q-115 74 -178.5 74t-113.5 -45.5t-69 -90.5l-18 -45z" />
-<glyph unicode="&#xa1;" horiz-adv-x="387" d="M74 -385l55 1100h129l55 -1100h-239zM86 893v227h215v-227h-215z" />
-<glyph unicode="&#xa2;" horiz-adv-x="636" d="M66 508v489q0 297 208 328v242h123v-244q98 -16 144.5 -88t46.5 -227v-88h-189v135q0 90 -72.5 90t-72.5 -90v-604q0 -90 72 -91q74 0 73 91v155h189v-108q0 -156 -46 -228.5t-145 -89.5v-303h-123v301q-209 31 -208 330z" />
-<glyph unicode="&#xa3;" horiz-adv-x="817" d="M4 63q8 20 23.5 53.5t70 91.5t117.5 68q37 111 37 189t-31 184h-188v137h147l-6 21q-78 254 -78 333t15.5 140t48.5 116q72 122 231 126q190 4 267 -126q65 -108 65 -276h-213q0 201 -115 197q-47 -2 -68.5 -51t-21.5 -139.5t70 -315.5l6 -25h211v-137h-174 q25 -100 24.5 -189t-57.5 -204q16 -8 44 -24q59 -35 89 -35q74 4 82 190l188 -22q-12 -182 -81.5 -281.5t-169.5 -99.5q-51 0 -143.5 51t-127.5 51t-63.5 -25.5t-40.5 -52.5l-12 -24z" />
-<glyph unicode="&#xa5;" horiz-adv-x="720" d="M25 1505h217l110 -481l6 -14h4l7 14l110 481h217l-196 -753h147v-138h-176v-137h176v-137h-176v-340h-221v340h-176v137h176v137h-176v138h147z" />
-<glyph unicode="&#xa8;" horiz-adv-x="1024" d="M272 1305v200h191v-200h-191zM561 1305v200h191v-200h-191z" />
-<glyph unicode="&#xa9;" horiz-adv-x="1644" d="M53 751.5q0 317.5 225.5 544t543 226.5t543.5 -226.5t226 -544t-226 -542.5t-543.5 -225t-543 225t-225.5 542.5zM172 751.5q0 -266.5 191.5 -458t457.5 -191.5t459 191.5t193 459t-191.5 459t-459 191.5t-459 -192.5t-191.5 -459zM627 487v531q0 122 97 174q40 22 95 22 q147 0 182 -147l7 -49v-125h-138v142q0 11 -12 28.5t-37 17.5q-47 -2 -49 -63v-531q0 -63 49 -63q53 2 49 63v125h138v-125q0 -68 -40 -127q-18 -26 -57 -47.5t-108.5 -21.5t-117.5 49t-54 98z" />
-<glyph unicode="&#xaa;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM252 291 q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250z" />
-<glyph unicode="&#xad;" horiz-adv-x="444" d="M74 455v194h297v-194h-297z" />
-<glyph unicode="&#xae;" horiz-adv-x="1644" d="M53 751.5q0 317.5 225.5 544t543 226.5t543.5 -226.5t226 -544t-226 -542.5t-543.5 -225t-543 225t-225.5 542.5zM172 751.5q0 -266.5 191.5 -458t457.5 -191.5t459 191.5t193 459t-191.5 459t-459 191.5t-459 -192.5t-191.5 -459zM625 313v879h196q231 0 232 -258 q0 -76 -16.5 -125t-71.5 -96l106 -400h-151l-95 365h-55v-365h-145zM770 805h45q43 0 65.5 21.5t27.5 45t5 61.5t-5 62.5t-27.5 46t-65.5 21.5h-45v-258z" />
-<glyph unicode="&#xaf;" horiz-adv-x="1024" d="M313 1315v162h398v-162h-398z" />
-<glyph unicode="&#xb2;" horiz-adv-x="731" d="M55 0v219l39 62q25 39 88.5 152.5t112.5 220t91 241.5t44 238q0 184 -73.5 184t-73.5 -184v-105h-222v105q0 389 295 389t295 -375q0 -336 -346 -928h350v-219h-600z" />
-<glyph unicode="&#xb3;" horiz-adv-x="686" d="M45 1071q0 249 63 343q29 42 84.5 75t134.5 33t136 -31t84.5 -71t44.5 -92q22 -71 22 -130q0 -291 -108 -399q127 -100 127 -414q0 -68 -19.5 -145.5t-47 -128t-85 -89t-136.5 -38.5t-135 31.5t-86 75.5t-48 113q-23 91 -23 230h217q2 -150 17.5 -203t59.5 -53t56.5 50.5 t12.5 104.5t1 102t0 63q-6 82 -14 95l-18 33q-12 22 -29 29q-55 22 -108 25h-19v184q133 7 156 73q12 34 12 91v105q0 146 -29 177q-16 17 -40 17q-41 0 -52.5 -49t-13.5 -207h-217z" />
-<glyph unicode="&#xb4;" horiz-adv-x="1024" d="M410 1305l106 184h215l-162 -184h-159z" />
-<glyph unicode="&#xb7;" horiz-adv-x="215" d="M0 649v228h215v-228h-215z" />
-<glyph unicode="&#xb8;" horiz-adv-x="1024" d="M426 -111h172v-141l-45 -133h-104l40 133h-63v141z" />
-<glyph unicode="&#xb9;" horiz-adv-x="475" d="M25 1180v141q129 25 205 130q16 21 30 54h133v-1505h-221v1180h-147z" />
-<glyph unicode="&#xba;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM257 259q0 -17 9 -44q18 -49 62 -49q70 10 71 113v563l1 19q0 19 -10 45q-18 50 -62 50 q-68 -10 -70 -114v-563q1 -1 1 -4z" />
-<glyph unicode="&#xbf;" horiz-adv-x="645" d="M41 -106q0 82 80 219l57 95q18 32 42 106.5t24 144.5v256h190v-256q0 -102 -24.5 -195.5t-48 -140.5t-65.5 -118t-50 -104.5t9 -67.5t60 -35t78 48.5t49 98.5l179 -84q-24 -66 -78 -132q-104 -126 -236 -122q-163 4 -220 115q-46 90 -46 172zM231 893v227h215v-227h-215z " />
-<glyph unicode="&#xc0;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM141 1823h215l107 -185h-160zM307 541h152l-64 475l-6 39h-12z" />
-<glyph unicode="&#xc1;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM293 1638l106 185h215l-161 -185h-160zM307 541h152l-64 475l-6 39h-12z" />
-<glyph unicode="&#xc2;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM133 1638l141 185h220l141 -185h-189l-63 72l-61 -72h-189zM307 541h152l-64 475l-6 39h-12z" />
-<glyph unicode="&#xc3;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM184 1632v152q49 39 95.5 39t104.5 -18.5t100.5 -19.5t97.5 32v-152q-51 -39 -95.5 -39t-102.5 19.5t-98 19.5t-102 -33zM307 541h152l-64 475l-6 39h-12z" />
-<glyph unicode="&#xc4;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM143 1638v201h191v-201h-191zM307 541h152l-64 475l-6 39h-12zM432 1638v201h191v-201h-191z" />
-<glyph unicode="&#xc5;" horiz-adv-x="765" d="M20 0l228 1505h270l227 -1505h-215l-41 307h-213l-40 -307h-216zM231 1761.5q0 61.5 45.5 102.5t109 41t107.5 -41t44 -102.5t-44 -102.5t-107.5 -41t-109 41t-45.5 102.5zM307 541h152l-64 475l-6 39h-12zM309 1761.5q0 -28.5 23.5 -50t52.5 -21.5t52.5 21.5t23.5 50 t-23.5 50t-52.5 21.5t-52.5 -21.5t-23.5 -50z" />
-<glyph unicode="&#xc6;" horiz-adv-x="1099" d="M16 0l420 1505h623v-227h-285v-395h205v-242h-205v-414h285v-227h-506v307h-227l-90 -307h-220zM393 541h160v514h-10z" />
-<glyph unicode="&#xc7;" horiz-adv-x="708" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-207h-206v207q-2 0 0 11.5t-3.5 27.5t-12.5 33q-17 39 -68 39q-70 -10 -78 -111v-887q0 -43 21.5 -76.5t59.5 -33.5t59.5 27.5t21.5 56.5v233h206v-207q0 -42 -17 -106t-45 -107 t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM268 -111v-141h64l-41 -133h104l45 133v141h-172z" />
-<glyph unicode="&#xc8;" horiz-adv-x="628" d="M82 0v1505h506v-227h-285v-395h205v-242h-205v-414h285v-227h-506zM111 1823h215l106 -185h-160z" />
-<glyph unicode="&#xc9;" horiz-adv-x="628" d="M82 0v1505h506v-227h-285v-395h205v-242h-205v-414h285v-227h-506zM236 1638l106 185h215l-162 -185h-159z" />
-<glyph unicode="&#xca;" horiz-adv-x="628" d="M82 0v1505h506v-227h-285v-395h205v-242h-205v-414h285v-227h-506zM84 1638l141 185h219l142 -185h-189l-63 72l-62 -72h-188z" />
-<glyph unicode="&#xcb;" horiz-adv-x="628" d="M82 0v1505h506v-227h-285v-395h205v-242h-205v-414h285v-227h-506zM94 1638v201h191v-201h-191zM383 1638v201h190v-201h-190z" />
-<glyph unicode="&#xcc;" horiz-adv-x="401" d="M-6 1823h215l106 -185h-159zM98 0v1505h221v-1505h-221z" />
-<glyph unicode="&#xcd;" horiz-adv-x="401" d="M82 0v1505h221v-1505h-221zM86 1638l107 185h215l-162 -185h-160z" />
-<glyph unicode="&#xce;" horiz-adv-x="370" d="M-66 1638l142 185h219l141 -185h-188l-64 72l-61 -72h-189zM74 0v1505h221v-1505h-221z" />
-<glyph unicode="&#xcf;" horiz-adv-x="372" d="M-53 1638v201h190v-201h-190zM76 0v1505h221v-1505h-221zM236 1638v201h190v-201h-190z" />
-<glyph unicode="&#xd0;" horiz-adv-x="761" d="M20 655v228h62v622h174q270 0 346 -113q31 -46 50.5 -95.5t28.5 -139.5t12 -177t3 -228.5t-3 -228.5t-12 -176t-28.5 -138t-50.5 -95t-80 -68q-106 -46 -266 -46h-174v655h-62zM303 221q117 0 141.5 81t22.5 452q2 371 -22.5 450.5t-141.5 79.5v-401h84v-228h-84v-434z " />
-<glyph unicode="&#xd1;" horiz-adv-x="808" d="M82 0v1505h197l215 -784l18 -70h12v854h203v-1505h-197l-215 784l-18 70h-12v-854h-203zM207 1632v152q49 39 95 39t104.5 -18.5t102.5 -19.5t95 32v-152q-51 -39 -95 -39t-102.5 19.5t-100 19.5t-99.5 -33z" />
-<glyph unicode="&#xd2;" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM121 1823h215l106 -185h-159zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5 l2 26v887q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="&#xd3;" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM285 1638l106 185h215l-162 -185h-159zM289 309q0 -46 19.5 -78t54 -32t53 27.5 t18.5 56.5l2 26v887q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="&#xd4;" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM113 1638l141 185h219l141 -185h-188l-64 72l-61 -72h-188zM289 309q0 -46 19.5 -78 t54 -32t53 27.5t18.5 56.5l2 26v887q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="&#xd5;" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM164 1632v152q49 39 95 39t104.5 -18.5t102.5 -19.5t95 32v-152q-51 -39 -95 -39 t-102.5 19.5t-100 19.5t-99.5 -33zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5l2 26v887q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887z" />
-<glyph unicode="&#xd6;" d="M68 309v887q0 42 17 106t45 107t88.5 78t144 35t144 -34t88.5 -81q55 -93 60 -178l2 -33v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-144 -34.5t-144 33.5t-88.5 81.5q-55 94 -60 175zM123 1638v201h190v-201h-190zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5 l2 26v887q0 46 -19.5 78.5t-54 32.5t-53 -28t-18.5 -54l-2 -29v-887zM412 1638v201h190v-201h-190z" />
-<glyph unicode="&#xd8;" d="M59 -20l47 157q-36 74 -36 148l-2 24v887q0 42 17 106t45 107t88.5 78t148 35t153.5 -43l15 47h122l-45 -150q43 -84 43 -155l2 -25v-887q0 -42 -17 -106t-45 -107t-88.5 -77.5t-150.5 -34.5t-153 43l-15 -47h-129zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5 l2 26v488zM289 727l147 479q-8 100 -74 101q-35 0 -53 -28t-18 -54l-2 -29v-469z" />
-<glyph unicode="&#xd9;" horiz-adv-x="749" d="M80 309q0 -42 17.5 -106t45 -107t88 -77.5t144.5 -34.5t144.5 33.5t88.5 81.5q55 97 60 175l2 35v1196h-221v-1196q0 -44 -19.5 -77t-54.5 -33t-53.5 27.5t-18.5 56.5l-2 26v1196h-221v-1196zM145 1823h215l107 -185h-160z" />
-<glyph unicode="&#xda;" horiz-adv-x="749" d="M80 309q0 -42 17.5 -106t45 -107t88 -77.5t144.5 -34.5t144.5 33.5t88.5 81.5q55 97 60 175l2 35v1196h-221v-1196q0 -44 -19.5 -77t-54.5 -33t-53.5 27.5t-18.5 56.5l-2 26v1196h-221v-1196zM307 1638l107 185h215l-162 -185h-160z" />
-<glyph unicode="&#xdb;" horiz-adv-x="749" d="M80 309q0 -42 17.5 -106t45 -107t88 -77.5t144.5 -34.5t144.5 33.5t88.5 81.5q55 97 60 175l2 35v1196h-221v-1196q0 -44 -19.5 -77t-54.5 -33t-53.5 27.5t-18.5 56.5l-2 26v1196h-221v-1196zM125 1638l141 185h219l142 -185h-189l-63 72l-62 -72h-188z" />
-<glyph unicode="&#xdc;" horiz-adv-x="749" d="M80 309v1196h221v-1196q0 -46 19.5 -78t54.5 -32t53 27.5t18 56.5l3 26v1196h221v-1196q0 -42 -17.5 -106t-45 -107t-88 -77.5t-144.5 -34.5t-144.5 33.5t-88.5 81.5q-55 97 -60 175zM135 1638v201h191v-201h-191zM424 1638v201h190v-201h-190z" />
-<glyph unicode="&#xdd;" horiz-adv-x="704" d="M16 1505l226 -864v-641h221v641l225 864h-217l-111 -481l-6 -14h-4l-6 14l-111 481h-217zM254 1638l106 185h215l-161 -185h-160z" />
-<glyph unicode="&#xde;" d="M82 0v1505h219v-241h2q166 0 277.5 -105.5t111.5 -345.5t-111.5 -346.5t-277.5 -106.5v-360h-221zM303 586q102 0 134 45t32 175t-33 181t-133 51v-452z" />
-<glyph unicode="&#xdf;" horiz-adv-x="733" d="M66 0v1235q0 123 70.5 205t206.5 82t204.5 -81t68.5 -197t-88 -181q152 -88 152 -488q0 -362 -87 -475q-46 -59 -102.5 -79.5t-144.5 -20.5v193q45 0 70 25q57 57 57 357q0 316 -57 377q-25 27 -70 27v141q35 0 60.5 33t25.5 84q0 100 -86 100q-74 0 -74 -102v-1235h-206 z" />
-<glyph unicode="&#xe0;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM102 1489h215 l107 -184h-160zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250z" />
-<glyph unicode="&#xe1;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM252 291 q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250zM264 1305l107 184h215l-162 -184h-160z" />
-<glyph unicode="&#xe2;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM90 1305 l141 184h220l141 -184h-189l-63 71l-61 -71h-189zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250z" />
-<glyph unicode="&#xe3;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM143 1305v151 q49 39 95.5 39t104.5 -18.5t97 -19.5t101 32v-152q-51 -39 -95.5 -39t-102.5 19.5t-99 19.5t-101 -32zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250z" />
-<glyph unicode="&#xe4;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM102 1305v200 h191v-200h-191zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250zM391 1305v200h191v-200h-191z" />
-<glyph unicode="&#xe5;" horiz-adv-x="681" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t190 88t184.5 -74t75 -180v-688q0 -109 14 -195h-202q-18 20 -19 90h-14q-20 -37 -65.5 -71.5t-102.5 -34.5t-110.5 60t-53.5 191zM188 1421.5 q0 61.5 45.5 102.5t109 41t107.5 -41t44 -102.5t-44 -102.5t-107.5 -41t-109 41t-45.5 102.5zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250zM266 1421.5q0 -28.5 23.5 -50t52.5 -21.5t52.5 21.5t23.5 50t-23.5 50t-52.5 21.5t-52.5 -21.5 t-23.5 -50z" />
-<glyph unicode="&#xe6;" horiz-adv-x="989" d="M49 235q0 131 34 212t83 124t98 73t88 50.5t43 36.5v123q0 102 -57 102q-41 0 -50 -42t-9 -84v-39h-207v47q0 123 80.5 211t197.5 88q84 0 152 -52q66 51 162 52q199 0 251 -197q14 -51 15 -92v-326h-342v-256q0 -60 38 -88q17 -12 38 -12q70 10 73 113v122h193v-129 q0 -37 -16.5 -93t-41 -95t-80 -69.5t-130.5 -30.5q-158 0 -226 131q-102 -131 -221 -131q-59 0 -112.5 60t-53.5 191zM252 291q0 -104 57 -105q35 0 60.5 19.5t25.5 48.5v287q-143 -62 -143 -250zM588 684h149v158q0 48 -19.5 81t-53 33t-53 -28.5t-21.5 -57.5l-2 -28v-158z " />
-<glyph unicode="&#xe7;" horiz-adv-x="645" d="M66 315v490q0 332 264 332q137 0 201.5 -71t64.5 -251v-88h-207v135q0 51 -12 70.5t-47 19.5q-58 0 -58 -90v-604q0 -90 58 -90q35 0 47 19.5t12 70.5v156h207v-109q0 -180 -64.5 -250.5t-201.5 -70.5q-264 0 -264 331zM238 -111v-141h63l-41 -133h105l45 133v141h-172z " />
-<glyph unicode="&#xe8;" horiz-adv-x="659" d="M66 279v563q0 36 16 94.5t42 97.5t81 71t129 32q199 0 252 -197q14 -51 14 -92v-326h-342v-256q0 -59 39 -88q16 -12 37 -12q70 10 74 113v122h192v-129q0 -37 -16.5 -93t-41 -95t-79.5 -69.5t-130 -30.5t-130.5 30.5t-80.5 73.5q-49 87 -54 160zM102 1489h215l107 -184 h-160zM258 684h150v158q0 48 -19.5 81t-53.5 33t-53.5 -28.5t-21.5 -57.5l-2 -28v-158z" />
-<glyph unicode="&#xe9;" horiz-adv-x="659" d="M66 279v563q0 36 16 94.5t42 97.5t81 71t129 32q199 0 252 -197q14 -51 14 -92v-326h-342v-256q0 -59 39 -88q16 -12 37 -12q70 10 74 113v122h192v-129q0 -37 -16.5 -93t-41 -95t-79.5 -69.5t-130 -30.5t-130.5 30.5t-80.5 73.5q-49 87 -54 160zM258 684h150v158 q0 48 -19.5 81t-53.5 33t-53.5 -28.5t-21.5 -57.5l-2 -28v-158zM264 1305l107 184h215l-162 -184h-160z" />
-<glyph unicode="&#xea;" horiz-adv-x="659" d="M66 279v563q0 36 16 94.5t42 97.5t81 71t129 32q199 0 252 -197q14 -51 14 -92v-326h-342v-256q0 -59 39 -88q16 -12 37 -12q70 10 74 113v122h192v-129q0 -37 -16.5 -93t-41 -95t-79.5 -69.5t-130 -30.5t-130.5 30.5t-80.5 73.5q-49 87 -54 160zM80 1305l141 184h219 l142 -184h-189l-63 71l-62 -71h-188zM258 684h150v158q0 48 -19.5 81t-53.5 33t-53.5 -28.5t-21.5 -57.5l-2 -28v-158z" />
-<glyph unicode="&#xeb;" horiz-adv-x="659" d="M66 279v563q0 36 16 94.5t42 97.5t81 71t129 32q199 0 252 -197q14 -51 14 -92v-326h-342v-256q0 -59 39 -88q16 -12 37 -12q70 10 74 113v122h192v-129q0 -37 -16.5 -93t-41 -95t-79.5 -69.5t-130 -30.5t-130.5 30.5t-80.5 73.5q-49 87 -54 160zM90 1305v200h191v-200 h-191zM258 684h150v158q0 48 -19.5 81t-53.5 33t-53.5 -28.5t-21.5 -57.5l-2 -28v-158zM379 1305v200h190v-200h-190z" />
-<glyph unicode="&#xec;" horiz-adv-x="370" d="M-33 1489h215l107 -184h-160zM82 0h207v1120h-207v-1120z" />
-<glyph unicode="&#xed;" horiz-adv-x="370" d="M82 0h207v1120h-207v-1120zM82 1305l106 184h215l-161 -184h-160z" />
-<glyph unicode="&#xee;" horiz-adv-x="370" d="M-66 1305l142 184h219l141 -184h-188l-64 71l-61 -71h-189zM82 0h207v1120h-207v-1120z" />
-<glyph unicode="&#xef;" horiz-adv-x="372" d="M-53 1305v200h190v-200h-190zM82 0v1120h207v-1120h-207zM236 1305v200h190v-200h-190z" />
-<glyph unicode="&#xf0;" horiz-adv-x="673" d="M76 279v579q0 279 172 279q63 0 155 -78q-12 109 -51 203l-82 -72l-55 63l100 88l-45 66l109 100q25 -27 53 -61l94 82l56 -66l-101 -88q125 -201 125 -446v-656q0 -102 -56 -188q-26 -39 -80 -69.5t-129 -30.5t-130 30.5t-80 73.5q-53 91 -53 160zM270 267.5 q-2 -11.5 2 -29t10 -34.5q16 -38 58 -38q70 10 72 113v563q-2 0 0 11t-2 28.5t-10 34.5q-16 40 -60 40q-68 -10 -70 -114v-563q2 0 0 -11.5z" />
-<glyph unicode="&#xf1;" horiz-adv-x="690" d="M82 0v1120h207v-94l32 32q79 79 145.5 79t106 -69t39.5 -150v-918h-206v887q-1 49 -50 49q-41 0 -67 -53v-883h-207zM147 1305v151q49 39 95.5 39t105 -18.5t97 -19.5t100.5 32v-152q-51 -39 -95.5 -39t-102.5 19.5t-99 19.5t-101 -32z" />
-<glyph unicode="&#xf2;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM98 1489h215l107 -184h-160zM258 267.5q-2 -11.5 2 -29t10 -34.5q14 -38 58 -38 q70 10 71 113v563q-2 0 0 11t-2 28.5t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5z" />
-<glyph unicode="&#xf3;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM258 267.5q-2 -11.5 2 -29t10 -34.5q14 -38 58 -38q70 10 71 113v563q-2 0 0 11t-2 28.5 t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5zM260 1305l107 184h215l-162 -184h-160z" />
-<glyph unicode="&#xf4;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM78 1305l141 184h219l142 -184h-189l-63 71l-62 -71h-188zM258 267.5q-2 -11.5 2 -29 t10 -34.5q14 -38 58 -38q70 10 71 113v563q-2 0 0 11t-2 28.5t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5z" />
-<glyph unicode="&#xf5;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM131 1305v151q49 39 95.5 39t104.5 -18.5t98.5 -19.5t98.5 32v-152q-51 -39 -95 -39 t-102 19.5t-101 19.5t-99 -32zM258 267.5q-2 -11.5 2 -29t10 -34.5q14 -38 58 -38q70 10 71 113v563q-2 0 0 11t-2 28.5t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5z" />
-<glyph unicode="&#xf6;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t129.5 32q199 0 252 -197q14 -51 14 -92v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-129 -30.5t-130 30.5t-80.5 73.5q-52 92 -52 160zM90 1305v200h191v-200h-191zM258 267.5q-2 -11.5 2 -29t10 -34.5q14 -38 58 -38 q70 10 71 113v563q-2 0 0 11t-2 28.5t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5zM379 1305v200h190v-200h-190z" />
-<glyph unicode="&#xf8;" horiz-adv-x="657" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t118 32t117.5 -19l21 80h75l-30 -121q88 -84 94 -229v-576q0 -102 -56 -188q-26 -39 -80.5 -69.5t-120.5 -30.5t-112 16l-20 -78h-80l31 121q-41 39 -64.5 97.5t-25.5 97.5zM258 436l125 486q-18 35 -55 34q-68 -10 -70 -114 v-406zM274 197q17 -31 54 -31q70 10 71 113v403z" />
-<glyph unicode="&#xf9;" horiz-adv-x="690" d="M78 203v917h207v-887q0 -49 49 -49q41 0 67 54v882h207v-1120h-207v94l-31 -32q-78 -78 -145.5 -78t-107 68.5t-39.5 150.5zM113 1489h215l106 -184h-160z" />
-<glyph unicode="&#xfa;" horiz-adv-x="690" d="M78 203v917h207v-887q0 -49 49 -49q41 0 67 54v882h207v-1120h-207v94l-31 -32q-78 -78 -145.5 -78t-107 68.5t-39.5 150.5zM274 1305l107 184h215l-162 -184h-160z" />
-<glyph unicode="&#xfb;" horiz-adv-x="690" d="M78 203v917h207v-887q0 -49 49 -49q41 0 67 54v882h207v-1120h-207v94l-31 -32q-78 -78 -145.5 -78t-107 68.5t-39.5 150.5zM94 1305l142 184h219l141 -184h-188l-64 71l-61 -71h-189z" />
-<glyph unicode="&#xfc;" horiz-adv-x="690" d="M78 203v917h207v-887q0 -49 49 -49q41 0 67 54v882h207v-1120h-207v94l-31 -32q-78 -78 -145.5 -78t-107 68.5t-39.5 150.5zM106 1305v200h191v-200h-191zM395 1305v200h191v-200h-191z" />
-<glyph unicode="&#xfd;" horiz-adv-x="634" d="M25 1120l190 -1153q0 -68 -36 -123t-97 -61l-49 4v-184q70 -4 92 -4q115 0 192.5 95t94.5 222l198 1204h-202l-82 -688l-4 -57h-9l-4 57l-82 688h-202zM231 1305l107 184h215l-162 -184h-160z" />
-<glyph unicode="&#xfe;" horiz-adv-x="686" d="M82 -385v1890h207v-458q88 90 165 90t117.5 -69t40.5 -150v-715q0 -82 -41 -150.5t-118 -68.5q-33 0 -74 22.5t-66 44.5l-24 23v-459h-207zM289 246q0 -25 19.5 -46.5t42 -21.5t39 19.5t16.5 48.5v628q0 29 -16.5 48.5t-39 19.5t-42 -19.5t-19.5 -48.5v-628z" />
-<glyph unicode="&#xff;" horiz-adv-x="634" d="M25 1120h202l82 -688l4 -57h9l4 57l82 688h202l-198 -1204q-16 -127 -94 -222t-193 -95l-92 4v184q16 -4 49 -4q61 6 97 61.5t36 122.5zM78 1305v200h190v-200h-190zM367 1305v200h190v-200h-190z" />
-<glyph unicode="&#x152;" horiz-adv-x="983" d="M68 309v887q0 41 17 101.5t45 100.5t88.5 73.5t143.5 33.5h580v-227h-285v-395h205v-242h-205v-414h285v-227h-580q-84 0 -144 31.5t-88 78.5q-55 91 -60 169zM289 309q0 -46 19.5 -78t54 -32t53 27.5t18.5 56.5l2 26v901q-6 96 -74 97q-35 0 -53 -28t-18 -54l-2 -29 v-887z" />
-<glyph unicode="&#x153;" horiz-adv-x="995" d="M63 279v563q0 40 15.5 96.5t40 95.5t80 71t145.5 32t156 -60q66 59 170 60q199 0 252 -197q14 -51 14 -92v-326h-342v-250q0 -46 22.5 -76t53.5 -30q70 10 73 113v122h193v-129q0 -37 -16.5 -93t-41 -95t-80 -69.5t-146 -30.5t-154.5 57q-68 -57 -156 -57t-143.5 30.5 t-80.5 73.5q-52 92 -52 160zM258 267.5q-2 -11.5 2 -29t10 -34.5q14 -38 58 -38q70 10 71 113v563q-2 0 0 11t-2 28.5t-10 34.5q-15 40 -59 40q-68 -10 -70 -114v-563q2 0 0 -11.5zM594 684h149v158q0 48 -19 81t-58 33t-55.5 -37.5t-16.5 -70.5v-164z" />
-<glyph unicode="&#x178;" horiz-adv-x="704" d="M16 1505h217l111 -481l6 -14h4l6 14l111 481h217l-225 -864v-641h-221v641zM113 1638v201h190v-201h-190zM401 1638v201h191v-201h-191z" />
-<glyph unicode="&#x2c6;" horiz-adv-x="1021" d="M260 1305l141 184h220l141 -184h-189l-63 71l-61 -71h-189z" />
-<glyph unicode="&#x2dc;" horiz-adv-x="1024" d="M313 1305v151q49 39 95.5 39t104.5 -18.5t97 -19.5t101 32v-152q-51 -39 -95.5 -39t-102.5 19.5t-99 19.5t-101 -32z" />
-<glyph unicode="&#x2000;" horiz-adv-x="952" />
-<glyph unicode="&#x2001;" horiz-adv-x="1905" />
-<glyph unicode="&#x2002;" horiz-adv-x="952" />
-<glyph unicode="&#x2003;" horiz-adv-x="1905" />
-<glyph unicode="&#x2004;" horiz-adv-x="635" />
-<glyph unicode="&#x2005;" horiz-adv-x="476" />
-<glyph unicode="&#x2006;" horiz-adv-x="317" />
-<glyph unicode="&#x2007;" horiz-adv-x="317" />
-<glyph unicode="&#x2008;" horiz-adv-x="238" />
-<glyph unicode="&#x2009;" horiz-adv-x="381" />
-<glyph unicode="&#x200a;" horiz-adv-x="105" />
-<glyph unicode="&#x2010;" horiz-adv-x="444" d="M74 455v194h297v-194h-297z" />
-<glyph unicode="&#x2011;" horiz-adv-x="444" d="M74 455v194h297v-194h-297z" />
-<glyph unicode="&#x2012;" horiz-adv-x="444" d="M74 455v194h297v-194h-297z" />
-<glyph unicode="&#x2013;" horiz-adv-x="806" d="M74 649v195h659v-195h-659z" />
-<glyph unicode="&#x2014;" horiz-adv-x="972" d="M74 649v195h825v-195h-825z" />
-<glyph unicode="&#x2018;" horiz-adv-x="309" d="M49 1012v227l113 266h102l-71 -266h71v-227h-215z" />
-<glyph unicode="&#x2019;" horiz-adv-x="309" d="M45 1012l72 266h-72v227h215v-227l-113 -266h-102z" />
-<glyph unicode="&#x201a;" horiz-adv-x="309" d="M45 0v227h215v-227l-113 -266h-102l72 266h-72z" />
-<glyph unicode="&#x201c;" horiz-adv-x="624" d="M53 1012v227l113 266h102l-71 -266h71v-227h-215zM356 1012v227l113 266h102l-71 -266h71v-227h-215z" />
-<glyph unicode="&#x201d;" horiz-adv-x="624" d="M53 1012l72 266h-72v227h215v-227l-112 -266h-103zM356 1012l72 266h-72v227h215v-227l-112 -266h-103z" />
-<glyph unicode="&#x201e;" horiz-adv-x="624" d="M53 0v227h215v-227l-112 -266h-103l72 266h-72zM356 0v227h215v-227l-112 -266h-103l72 266h-72z" />
-<glyph unicode="&#x2022;" horiz-adv-x="663" d="M82 815q0 104 72.5 177t177 73t177.5 -72.5t73 -177t-73 -177.5t-177 -73t-177 73t-73 177z" />
-<glyph unicode="&#x2026;" horiz-adv-x="964" d="M53 0v227h215v-227h-215zM375 0v227h215v-227h-215zM696 0v227h215v-227h-215z" />
-<glyph unicode="&#x202f;" horiz-adv-x="381" />
-<glyph unicode="&#x2039;" horiz-adv-x="1058" d="M74 649v160l911 475v-199l-698 -356l698 -356v-199z" />
-<glyph unicode="&#x203a;" horiz-adv-x="1058" d="M74 174v199l698 356l-698 356v199l911 -475v-160z" />
-<glyph unicode="&#x205f;" horiz-adv-x="476" />
-<glyph unicode="&#x20ac;" horiz-adv-x="813" d="M53 547v137h107v137h-107v137h107v238q0 42 17.5 106t45 107t88 78t144.5 35t144 -34t88 -81q53 -90 61 -178l2 -33v-84h-207v84q-2 0 0 11.5t-3 27.5t-12 33q-18 39 -69 39q-70 -10 -78 -111v-238h233v-137h-233v-137h233v-137h-233v-238q0 -43 21.5 -76.5t59.5 -33.5 t58.5 27.5t20.5 56.5l2 26v84h207v-84q0 -38 -17.5 -104t-45.5 -109t-88 -77.5t-144 -34.5t-144.5 33.5t-88.5 81.5q-55 97 -60 175l-2 35v238h-107z" />
-<glyph unicode="&#x2122;" horiz-adv-x="937" d="M74 1401v104h321v-104h-104v-580h-113v580h-104zM440 821v684h138l67 -319h6l68 319h137v-684h-104v449l-78 -449h-51l-80 449v-449h-103z" />
-<glyph unicode="&#xe000;" horiz-adv-x="1120" d="M0 0v1120h1120v-1120h-1120z" />
-<glyph unicode="&#xfb01;" horiz-adv-x="772" d="M20 934v186h105v31q0 172 31 231q16 31 42 67q53 71 181 71q59 0 127 -13l20 -2v-184q-41 12 -91 12t-69.5 -18.5t-25.5 -58.5q-8 -52 -8 -107v-29h358v-1120h-207v934h-151v-934h-207v934h-105z" />
-<glyph unicode="&#xfb02;" horiz-adv-x="772" d="M20 934v186h105v31q0 172 31 231q16 31 42 67q53 71 181 71q59 0 127 -13l20 -2h164v-1505h-207v1329q-37 4 -67.5 4t-50 -18.5t-25.5 -58.5q-8 -52 -8 -107v-29h104v-186h-104v-934h-207v934h-105z" />
-<glyph unicode="&#xfb03;" horiz-adv-x="1320" d="M20 934v186h105v31q0 190 51 270q23 35 71 63.5t115 28.5l97 -14v-178q-27 8 -62 8q-66 0 -65 -180v-29h104v-186h-104v-934h-207v934h-105zM495 934v186h105v31q0 190 51 270q23 35 71 63.5t115 28.5l97 -14v-178q-27 8 -62 8q-66 0 -65 -180v-29h104v-186h-104v-934 h-207v934h-105zM1032 0v1120h207v-1120h-207zM1032 1298v207h207v-207h-207z" />
-<glyph unicode="&#xfb04;" horiz-adv-x="1320" d="M20 934v186h105v31q0 190 51 270q23 35 71 63.5t115 28.5l97 -14v-178q-27 8 -62 8q-66 0 -65 -180v-29h104v-186h-104v-934h-207v934h-105zM495 934v186h105v31q0 190 51 270q23 35 71 63.5t115 28.5l97 -14v-178q-27 8 -62 8q-66 0 -65 -180v-29h104v-186h-104v-934 h-207v934h-105zM1032 0v1505h207v-1505h-207z" />
-</font>
-</defs></svg> 

website/source/assets/javascripts/lang-go.js

@@ -1,58 +0,0 @@
-// Copyright (C) 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
-
-/**
- * @fileoverview
- * Registers a language handler for the Go language..
- * <p>
- * Based on the lexical grammar at 
- * http://golang.org/doc/go_spec.html#Lexical_elements
- * <p>
- * Go uses a minimal style for highlighting so the below does not distinguish
- * strings, keywords, literals, etc. by design.
- * From a discussion with the Go designers:
- * <pre>
- * On Thursday, July 22, 2010, Mike Samuel <...> wrote:
- * > On Thu, Jul 22, 2010, Rob 'Commander' Pike <...> wrote:
- * >> Personally, I would vote for the subdued style godoc presents at http://golang.org
- * >>
- * >> Not as fancy as some like, but a case can be made it's the official style.
- * >> If people want more colors, I wouldn't fight too hard, in the interest of
- * >> encouragement through familiarity, but even then I would ask to shy away
- * >> from technicolor starbursts.
- * >
- * > Like http://golang.org/pkg/go/scanner/ where comments are blue and all
- * > other content is black?  I can do that.
- * </pre>
- *
- * @author mikesamuel@gmail.com
- */
-
-PR['registerLangHandler'](
-    PR['createSimpleLexer'](
-        [
-         // Whitespace is made up of spaces, tabs and newline characters.
-         [PR['PR_PLAIN'],       /^[\t\n\r \xA0]+/, null, '\t\n\r \xA0'],
-         // Not escaped as a string.  See note on minimalism above.
-         [PR['PR_PLAIN'],       /^(?:\"(?:[^\"\\]|\\[\s\S])*(?:\"|$)|\'(?:[^\'\\]|\\[\s\S])+(?:\'|$)|`[^`]*(?:`|$))/, null, '"\'']
-        ],
-        [
-         // Block comments are delimited by /* and */.
-         // Single-line comments begin with // and extend to the end of a line.
-         [PR['PR_COMMENT'],     /^(?:\/\/[^\r\n]*|\/\*[\s\S]*?\*\/)/],
-         [PR['PR_PLAIN'],       /^(?:[^\/\"\'`]|\/(?![\/\*]))+/i]
-        ]),
-    ['go']);

website/source/assets/javascripts/prettify.js

@@ -1,30 +0,0 @@
-!function(){var q=null;window.PR_SHOULD_USE_CONTINUATION=!0;
-(function(){function S(a){function d(e){var b=e.charCodeAt(0);if(b!==92)return b;var a=e.charAt(1);return(b=r[a])?b:"0"<=a&&a<="7"?parseInt(e.substring(1),8):a==="u"||a==="x"?parseInt(e.substring(2),16):e.charCodeAt(1)}function g(e){if(e<32)return(e<16?"\\x0":"\\x")+e.toString(16);e=String.fromCharCode(e);return e==="\\"||e==="-"||e==="]"||e==="^"?"\\"+e:e}function b(e){var b=e.substring(1,e.length-1).match(/\\u[\dA-Fa-f]{4}|\\x[\dA-Fa-f]{2}|\\[0-3][0-7]{0,2}|\\[0-7]{1,2}|\\[\S\s]|[^\\]/g),e=[],a=
-b[0]==="^",c=["["];a&&c.push("^");for(var a=a?1:0,f=b.length;a<f;++a){var h=b[a];if(/\\[bdsw]/i.test(h))c.push(h);else{var h=d(h),l;a+2<f&&"-"===b[a+1]?(l=d(b[a+2]),a+=2):l=h;e.push([h,l]);l<65||h>122||(l<65||h>90||e.push([Math.max(65,h)|32,Math.min(l,90)|32]),l<97||h>122||e.push([Math.max(97,h)&-33,Math.min(l,122)&-33]))}}e.sort(function(e,a){return e[0]-a[0]||a[1]-e[1]});b=[];f=[];for(a=0;a<e.length;++a)h=e[a],h[0]<=f[1]+1?f[1]=Math.max(f[1],h[1]):b.push(f=h);for(a=0;a<b.length;++a)h=b[a],c.push(g(h[0])),
-h[1]>h[0]&&(h[1]+1>h[0]&&c.push("-"),c.push(g(h[1])));c.push("]");return c.join("")}function s(e){for(var a=e.source.match(/\[(?:[^\\\]]|\\[\S\s])*]|\\u[\dA-Fa-f]{4}|\\x[\dA-Fa-f]{2}|\\\d+|\\[^\dux]|\(\?[!:=]|[()^]|[^()[\\^]+/g),c=a.length,d=[],f=0,h=0;f<c;++f){var l=a[f];l==="("?++h:"\\"===l.charAt(0)&&(l=+l.substring(1))&&(l<=h?d[l]=-1:a[f]=g(l))}for(f=1;f<d.length;++f)-1===d[f]&&(d[f]=++x);for(h=f=0;f<c;++f)l=a[f],l==="("?(++h,d[h]||(a[f]="(?:")):"\\"===l.charAt(0)&&(l=+l.substring(1))&&l<=h&&
-(a[f]="\\"+d[l]);for(f=0;f<c;++f)"^"===a[f]&&"^"!==a[f+1]&&(a[f]="");if(e.ignoreCase&&m)for(f=0;f<c;++f)l=a[f],e=l.charAt(0),l.length>=2&&e==="["?a[f]=b(l):e!=="\\"&&(a[f]=l.replace(/[A-Za-z]/g,function(a){a=a.charCodeAt(0);return"["+String.fromCharCode(a&-33,a|32)+"]"}));return a.join("")}for(var x=0,m=!1,j=!1,k=0,c=a.length;k<c;++k){var i=a[k];if(i.ignoreCase)j=!0;else if(/[a-z]/i.test(i.source.replace(/\\u[\da-f]{4}|\\x[\da-f]{2}|\\[^UXux]/gi,""))){m=!0;j=!1;break}}for(var r={b:8,t:9,n:10,v:11,
-f:12,r:13},n=[],k=0,c=a.length;k<c;++k){i=a[k];if(i.global||i.multiline)throw Error(""+i);n.push("(?:"+s(i)+")")}return RegExp(n.join("|"),j?"gi":"g")}function T(a,d){function g(a){var c=a.nodeType;if(c==1){if(!b.test(a.className)){for(c=a.firstChild;c;c=c.nextSibling)g(c);c=a.nodeName.toLowerCase();if("br"===c||"li"===c)s[j]="\n",m[j<<1]=x++,m[j++<<1|1]=a}}else if(c==3||c==4)c=a.nodeValue,c.length&&(c=d?c.replace(/\r\n?/g,"\n"):c.replace(/[\t\n\r ]+/g," "),s[j]=c,m[j<<1]=x,x+=c.length,m[j++<<1|1]=
-a)}var b=/(?:^|\s)nocode(?:\s|$)/,s=[],x=0,m=[],j=0;g(a);return{a:s.join("").replace(/\n$/,""),d:m}}function H(a,d,g,b){d&&(a={a:d,e:a},g(a),b.push.apply(b,a.g))}function U(a){for(var d=void 0,g=a.firstChild;g;g=g.nextSibling)var b=g.nodeType,d=b===1?d?a:g:b===3?V.test(g.nodeValue)?a:d:d;return d===a?void 0:d}function C(a,d){function g(a){for(var j=a.e,k=[j,"pln"],c=0,i=a.a.match(s)||[],r={},n=0,e=i.length;n<e;++n){var z=i[n],w=r[z],t=void 0,f;if(typeof w==="string")f=!1;else{var h=b[z.charAt(0)];
-if(h)t=z.match(h[1]),w=h[0];else{for(f=0;f<x;++f)if(h=d[f],t=z.match(h[1])){w=h[0];break}t||(w="pln")}if((f=w.length>=5&&"lang-"===w.substring(0,5))&&!(t&&typeof t[1]==="string"))f=!1,w="src";f||(r[z]=w)}h=c;c+=z.length;if(f){f=t[1];var l=z.indexOf(f),B=l+f.length;t[2]&&(B=z.length-t[2].length,l=B-f.length);w=w.substring(5);H(j+h,z.substring(0,l),g,k);H(j+h+l,f,I(w,f),k);H(j+h+B,z.substring(B),g,k)}else k.push(j+h,w)}a.g=k}var b={},s;(function(){for(var g=a.concat(d),j=[],k={},c=0,i=g.length;c<i;++c){var r=
-g[c],n=r[3];if(n)for(var e=n.length;--e>=0;)b[n.charAt(e)]=r;r=r[1];n=""+r;k.hasOwnProperty(n)||(j.push(r),k[n]=q)}j.push(/[\S\s]/);s=S(j)})();var x=d.length;return g}function v(a){var d=[],g=[];a.tripleQuotedStrings?d.push(["str",/^(?:'''(?:[^'\\]|\\[\S\s]|''?(?=[^']))*(?:'''|$)|"""(?:[^"\\]|\\[\S\s]|""?(?=[^"]))*(?:"""|$)|'(?:[^'\\]|\\[\S\s])*(?:'|$)|"(?:[^"\\]|\\[\S\s])*(?:"|$))/,q,"'\""]):a.multiLineStrings?d.push(["str",/^(?:'(?:[^'\\]|\\[\S\s])*(?:'|$)|"(?:[^"\\]|\\[\S\s])*(?:"|$)|`(?:[^\\`]|\\[\S\s])*(?:`|$))/,
-q,"'\"`"]):d.push(["str",/^(?:'(?:[^\n\r'\\]|\\.)*(?:'|$)|"(?:[^\n\r"\\]|\\.)*(?:"|$))/,q,"\"'"]);a.verbatimStrings&&g.push(["str",/^@"(?:[^"]|"")*(?:"|$)/,q]);var b=a.hashComments;b&&(a.cStyleComments?(b>1?d.push(["com",/^#(?:##(?:[^#]|#(?!##))*(?:###|$)|.*)/,q,"#"]):d.push(["com",/^#(?:(?:define|e(?:l|nd)if|else|error|ifn?def|include|line|pragma|undef|warning)\b|[^\n\r]*)/,q,"#"]),g.push(["str",/^<(?:(?:(?:\.\.\/)*|\/?)(?:[\w-]+(?:\/[\w-]+)+)?[\w-]+\.h(?:h|pp|\+\+)?|[a-z]\w*)>/,q])):d.push(["com",
-/^#[^\n\r]*/,q,"#"]));a.cStyleComments&&(g.push(["com",/^\/\/[^\n\r]*/,q]),g.push(["com",/^\/\*[\S\s]*?(?:\*\/|$)/,q]));if(b=a.regexLiterals){var s=(b=b>1?"":"\n\r")?".":"[\\S\\s]";g.push(["lang-regex",RegExp("^(?:^^\\.?|[+-]|[!=]=?=?|\\#|%=?|&&?=?|\\(|\\*=?|[+\\-]=|->|\\/=?|::?|<<?=?|>>?>?=?|,|;|\\?|@|\\[|~|{|\\^\\^?=?|\\|\\|?=?|break|case|continue|delete|do|else|finally|instanceof|return|throw|try|typeof)\\s*("+("/(?=[^/*"+b+"])(?:[^/\\x5B\\x5C"+b+"]|\\x5C"+s+"|\\x5B(?:[^\\x5C\\x5D"+b+"]|\\x5C"+
-s+")*(?:\\x5D|$))+/")+")")])}(b=a.types)&&g.push(["typ",b]);b=(""+a.keywords).replace(/^ | $/g,"");b.length&&g.push(["kwd",RegExp("^(?:"+b.replace(/[\s,]+/g,"|")+")\\b"),q]);d.push(["pln",/^\s+/,q," \r\n\t\u00a0"]);b="^.[^\\s\\w.$@'\"`/\\\\]*";a.regexLiterals&&(b+="(?!s*/)");g.push(["lit",/^@[$_a-z][\w$@]*/i,q],["typ",/^(?:[@_]?[A-Z]+[a-z][\w$@]*|\w+_t\b)/,q],["pln",/^[$_a-z][\w$@]*/i,q],["lit",/^(?:0x[\da-f]+|(?:\d(?:_\d+)*\d*(?:\.\d*)?|\.\d\+)(?:e[+-]?\d+)?)[a-z]*/i,q,"0123456789"],["pln",/^\\[\S\s]?/,
-q],["pun",RegExp(b),q]);return C(d,g)}function J(a,d,g){function b(a){var c=a.nodeType;if(c==1&&!x.test(a.className))if("br"===a.nodeName)s(a),a.parentNode&&a.parentNode.removeChild(a);else for(a=a.firstChild;a;a=a.nextSibling)b(a);else if((c==3||c==4)&&g){var d=a.nodeValue,i=d.match(m);if(i)c=d.substring(0,i.index),a.nodeValue=c,(d=d.substring(i.index+i[0].length))&&a.parentNode.insertBefore(j.createTextNode(d),a.nextSibling),s(a),c||a.parentNode.removeChild(a)}}function s(a){function b(a,c){var d=
-c?a.cloneNode(!1):a,e=a.parentNode;if(e){var e=b(e,1),g=a.nextSibling;e.appendChild(d);for(var i=g;i;i=g)g=i.nextSibling,e.appendChild(i)}return d}for(;!a.nextSibling;)if(a=a.parentNode,!a)return;for(var a=b(a.nextSibling,0),d;(d=a.parentNode)&&d.nodeType===1;)a=d;c.push(a)}for(var x=/(?:^|\s)nocode(?:\s|$)/,m=/\r\n?|\n/,j=a.ownerDocument,k=j.createElement("li");a.firstChild;)k.appendChild(a.firstChild);for(var c=[k],i=0;i<c.length;++i)b(c[i]);d===(d|0)&&c[0].setAttribute("value",d);var r=j.createElement("ol");
-r.className="linenums";for(var d=Math.max(0,d-1|0)||0,i=0,n=c.length;i<n;++i)k=c[i],k.className="L"+(i+d)%10,k.firstChild||k.appendChild(j.createTextNode("\u00a0")),r.appendChild(k);a.appendChild(r)}function p(a,d){for(var g=d.length;--g>=0;){var b=d[g];F.hasOwnProperty(b)?D.console&&console.warn("cannot override language handler %s",b):F[b]=a}}function I(a,d){if(!a||!F.hasOwnProperty(a))a=/^\s*</.test(d)?"default-markup":"default-code";return F[a]}function K(a){var d=a.h;try{var g=T(a.c,a.i),b=g.a;
-a.a=b;a.d=g.d;a.e=0;I(d,b)(a);var s=/\bMSIE\s(\d+)/.exec(navigator.userAgent),s=s&&+s[1]<=8,d=/\n/g,x=a.a,m=x.length,g=0,j=a.d,k=j.length,b=0,c=a.g,i=c.length,r=0;c[i]=m;var n,e;for(e=n=0;e<i;)c[e]!==c[e+2]?(c[n++]=c[e++],c[n++]=c[e++]):e+=2;i=n;for(e=n=0;e<i;){for(var p=c[e],w=c[e+1],t=e+2;t+2<=i&&c[t+1]===w;)t+=2;c[n++]=p;c[n++]=w;e=t}c.length=n;var f=a.c,h;if(f)h=f.style.display,f.style.display="none";try{for(;b<k;){var l=j[b+2]||m,B=c[r+2]||m,t=Math.min(l,B),A=j[b+1],G;if(A.nodeType!==1&&(G=x.substring(g,
-t))){s&&(G=G.replace(d,"\r"));A.nodeValue=G;var L=A.ownerDocument,o=L.createElement("span");o.className=c[r+1];var v=A.parentNode;v.replaceChild(o,A);o.appendChild(A);g<l&&(j[b+1]=A=L.createTextNode(x.substring(t,l)),v.insertBefore(A,o.nextSibling))}g=t;g>=l&&(b+=2);g>=B&&(r+=2)}}finally{if(f)f.style.display=h}}catch(u){D.console&&console.log(u&&u.stack||u)}}var D=window,y=["break,continue,do,else,for,if,return,while"],E=[[y,"auto,case,char,const,default,double,enum,extern,float,goto,inline,int,long,register,short,signed,sizeof,static,struct,switch,typedef,union,unsigned,void,volatile"],
-"catch,class,delete,false,import,new,operator,private,protected,public,this,throw,true,try,typeof"],M=[E,"alignof,align_union,asm,axiom,bool,concept,concept_map,const_cast,constexpr,decltype,delegate,dynamic_cast,explicit,export,friend,generic,late_check,mutable,namespace,nullptr,property,reinterpret_cast,static_assert,static_cast,template,typeid,typename,using,virtual,where"],N=[E,"abstract,assert,boolean,byte,extends,final,finally,implements,import,instanceof,interface,null,native,package,strictfp,super,synchronized,throws,transient"],
-O=[N,"as,base,by,checked,decimal,delegate,descending,dynamic,event,fixed,foreach,from,group,implicit,in,internal,into,is,let,lock,object,out,override,orderby,params,partial,readonly,ref,sbyte,sealed,stackalloc,string,select,uint,ulong,unchecked,unsafe,ushort,var,virtual,where"],E=[E,"debugger,eval,export,function,get,null,set,undefined,var,with,Infinity,NaN"],P=[y,"and,as,assert,class,def,del,elif,except,exec,finally,from,global,import,in,is,lambda,nonlocal,not,or,pass,print,raise,try,with,yield,False,True,None"],
-Q=[y,"alias,and,begin,case,class,def,defined,elsif,end,ensure,false,in,module,next,nil,not,or,redo,rescue,retry,self,super,then,true,undef,unless,until,when,yield,BEGIN,END"],W=[y,"as,assert,const,copy,drop,enum,extern,fail,false,fn,impl,let,log,loop,match,mod,move,mut,priv,pub,pure,ref,self,static,struct,true,trait,type,unsafe,use"],y=[y,"case,done,elif,esac,eval,fi,function,in,local,set,then,until"],R=/^(DIR|FILE|vector|(de|priority_)?queue|list|stack|(const_)?iterator|(multi)?(set|map)|bitset|u?(int|float)\d*)\b/,
-V=/\S/,X=v({keywords:[M,O,E,"caller,delete,die,do,dump,elsif,eval,exit,foreach,for,goto,if,import,last,local,my,next,no,our,print,package,redo,require,sub,undef,unless,until,use,wantarray,while,BEGIN,END",P,Q,y],hashComments:!0,cStyleComments:!0,multiLineStrings:!0,regexLiterals:!0}),F={};p(X,["default-code"]);p(C([],[["pln",/^[^<?]+/],["dec",/^<!\w[^>]*(?:>|$)/],["com",/^<\!--[\S\s]*?(?:--\>|$)/],["lang-",/^<\?([\S\s]+?)(?:\?>|$)/],["lang-",/^<%([\S\s]+?)(?:%>|$)/],["pun",/^(?:<[%?]|[%?]>)/],["lang-",
-/^<xmp\b[^>]*>([\S\s]+?)<\/xmp\b[^>]*>/i],["lang-js",/^<script\b[^>]*>([\S\s]*?)(<\/script\b[^>]*>)/i],["lang-css",/^<style\b[^>]*>([\S\s]*?)(<\/style\b[^>]*>)/i],["lang-in.tag",/^(<\/?[a-z][^<>]*>)/i]]),["default-markup","htm","html","mxml","xhtml","xml","xsl"]);p(C([["pln",/^\s+/,q," \t\r\n"],["atv",/^(?:"[^"]*"?|'[^']*'?)/,q,"\"'"]],[["tag",/^^<\/?[a-z](?:[\w-.:]*\w)?|\/?>$/i],["atn",/^(?!style[\s=]|on)[a-z](?:[\w:-]*\w)?/i],["lang-uq.val",/^=\s*([^\s"'>]*(?:[^\s"'/>]|\/(?=\s)))/],["pun",/^[/<->]+/],
-["lang-js",/^on\w+\s*=\s*"([^"]+)"/i],["lang-js",/^on\w+\s*=\s*'([^']+)'/i],["lang-js",/^on\w+\s*=\s*([^\s"'>]+)/i],["lang-css",/^style\s*=\s*"([^"]+)"/i],["lang-css",/^style\s*=\s*'([^']+)'/i],["lang-css",/^style\s*=\s*([^\s"'>]+)/i]]),["in.tag"]);p(C([],[["atv",/^[\S\s]+/]]),["uq.val"]);p(v({keywords:M,hashComments:!0,cStyleComments:!0,types:R}),["c","cc","cpp","cxx","cyc","m"]);p(v({keywords:"null,true,false"}),["json"]);p(v({keywords:O,hashComments:!0,cStyleComments:!0,verbatimStrings:!0,types:R}),
-["cs"]);p(v({keywords:N,cStyleComments:!0}),["java"]);p(v({keywords:y,hashComments:!0,multiLineStrings:!0}),["bash","bsh","csh","sh"]);p(v({keywords:P,hashComments:!0,multiLineStrings:!0,tripleQuotedStrings:!0}),["cv","py","python"]);p(v({keywords:"caller,delete,die,do,dump,elsif,eval,exit,foreach,for,goto,if,import,last,local,my,next,no,our,print,package,redo,require,sub,undef,unless,until,use,wantarray,while,BEGIN,END",hashComments:!0,multiLineStrings:!0,regexLiterals:2}),["perl","pl","pm"]);p(v({keywords:Q,
-hashComments:!0,multiLineStrings:!0,regexLiterals:!0}),["rb","ruby"]);p(v({keywords:E,cStyleComments:!0,regexLiterals:!0}),["javascript","js"]);p(v({keywords:"all,and,by,catch,class,else,extends,false,finally,for,if,in,is,isnt,loop,new,no,not,null,of,off,on,or,return,super,then,throw,true,try,unless,until,when,while,yes",hashComments:3,cStyleComments:!0,multilineStrings:!0,tripleQuotedStrings:!0,regexLiterals:!0}),["coffee"]);p(v({keywords:W,cStyleComments:!0,multilineStrings:!0}),["rc","rs","rust"]);
-p(C([],[["str",/^[\S\s]+/]]),["regex"]);var Y=D.PR={createSimpleLexer:C,registerLangHandler:p,sourceDecorator:v,PR_ATTRIB_NAME:"atn",PR_ATTRIB_VALUE:"atv",PR_COMMENT:"com",PR_DECLARATION:"dec",PR_KEYWORD:"kwd",PR_LITERAL:"lit",PR_NOCODE:"nocode",PR_PLAIN:"pln",PR_PUNCTUATION:"pun",PR_SOURCE:"src",PR_STRING:"str",PR_TAG:"tag",PR_TYPE:"typ",prettyPrintOne:D.prettyPrintOne=function(a,d,g){var b=document.createElement("div");b.innerHTML="<pre>"+a+"</pre>";b=b.firstChild;g&&J(b,g,!0);K({h:d,j:g,c:b,i:1});
-return b.innerHTML},prettyPrint:D.prettyPrint=function(a,d){function g(){for(var b=D.PR_SHOULD_USE_CONTINUATION?c.now()+250:Infinity;i<p.length&&c.now()<b;i++){for(var d=p[i],j=h,k=d;k=k.previousSibling;){var m=k.nodeType,o=(m===7||m===8)&&k.nodeValue;if(o?!/^\??prettify\b/.test(o):m!==3||/\S/.test(k.nodeValue))break;if(o){j={};o.replace(/\b(\w+)=([\w%+\-.:]+)/g,function(a,b,c){j[b]=c});break}}k=d.className;if((j!==h||e.test(k))&&!v.test(k)){m=!1;for(o=d.parentNode;o;o=o.parentNode)if(f.test(o.tagName)&&
-o.className&&e.test(o.className)){m=!0;break}if(!m){d.className+=" prettyprinted";m=j.lang;if(!m){var m=k.match(n),y;if(!m&&(y=U(d))&&t.test(y.tagName))m=y.className.match(n);m&&(m=m[1])}if(w.test(d.tagName))o=1;else var o=d.currentStyle,u=s.defaultView,o=(o=o?o.whiteSpace:u&&u.getComputedStyle?u.getComputedStyle(d,q).getPropertyValue("white-space"):0)&&"pre"===o.substring(0,3);u=j.linenums;if(!(u=u==="true"||+u))u=(u=k.match(/\blinenums\b(?::(\d+))?/))?u[1]&&u[1].length?+u[1]:!0:!1;u&&J(d,u,o);r=
-{h:m,c:d,j:u,i:o};K(r)}}}i<p.length?setTimeout(g,250):"function"===typeof a&&a()}for(var b=d||document.body,s=b.ownerDocument||document,b=[b.getElementsByTagName("pre"),b.getElementsByTagName("code"),b.getElementsByTagName("xmp")],p=[],m=0;m<b.length;++m)for(var j=0,k=b[m].length;j<k;++j)p.push(b[m][j]);var b=q,c=Date;c.now||(c={now:function(){return+new Date}});var i=0,r,n=/\blang(?:uage)?-([\w.]+)(?!\S)/,e=/\bprettyprint\b/,v=/\bprettyprinted\b/,w=/pre|xmp/i,t=/^code$/i,f=/^(?:pre|code|xmp)$/i,
-h={};g()}};typeof define==="function"&&define.amd&&define("google-code-prettify",[],function(){return Y})})();}()

website/source/assets/stylesheets/_docs.scss

@@ -159,6 +159,10 @@ body.layout-intro{
         -webkit-font-smoothing: antialiased;
 	}
 
+  li p a, li a {
+    text-decoration: none;
+  }
+
     pre{
         margin: 0 0 18px;
     }
@@ -192,6 +196,10 @@ body.layout-intro{
     #graph {
         margin-top: 30px;
     }
+
+  .alert p {
+    margin-bottom: 0;
+  }
 }
 
 

website/source/assets/stylesheets/_jumbotron.scss

@@ -16,6 +16,7 @@
   padding-bottom: 0;
   margin-top: $negative-hero-margin;
   color: $jumbotron-color;
+  -webkit-backface-visibility:hidden;
   @include consul-gradient-bg();
 
   &.mobile-hero{
@@ -40,7 +41,8 @@
       position: relative;
       height: 100%;
       margin-top: $header-height;
-
+      -webkit-backface-visibility:hidden;
+      
       .jumbo-logo-wrap{
         margin-top: 135px;