Utilise new raft.SetPeers() method, move expect logic to leader.go.

This way, we don't use EnableSingleMode, nor cause chaos adding peers.

Signed-off-by: Robert Xu <robxu9@gmail.com>

command/agent/command.go

@@ -137,7 +137,7 @@ func (c *Command) readConfig() *Config {
 
 	// Expect & Bootstrap are mutually exclusive
 	if config.Expect != 0 && config.Bootstrap {
-		c.Ui.Error("Expect mode and Bootstrap mode are mutually exclusive")
+		c.Ui.Error("Bootstrap cannot be provided with an expected server count")
 		return nil
 	}
 

consul/leader.go

@@ -369,57 +369,6 @@ func (s *Server) joinConsulServer(m serf.Member, parts *serverParts) error {
 		}
 	}
 
-	// Or, check for possibility that expect is not the same.
-	if parts.Expect != 0 {
-		members := s.serfLAN.Members()
-		for _, member := range members {
-			valid, p := isConsulServer(member)
-			if valid && member.Name != m.Name && p.Expect != parts.Expect {
-				s.logger.Printf("[ERR] consul: '%v' and '%v' have different expect values. All expect nodes should have the same value, not adding Raft peer.", m.Name, member.Name)
-				return nil
-			}
-		}
-	}
-
-	// If we're not a bootstrapped server, we're expecting servers,
-	// and our raft index is zero, try to auto bootstrap.
-	if !s.config.Bootstrap && s.config.Expect != 0 {
-		if index, _ := s.raftStore.LastIndex(); index == 0 {
-			// do not do standard op and add peer... yet
-			count := 0
-			members := s.serfLAN.Members()
-			for _, member := range members {
-				valid, p := isConsulServer(member)
-				if valid && member.Name != m.Name && p.Expect == parts.Expect {
-					count++
-					if count >= s.config.Expect {
-						break
-					}
-				}
-			}
-
-			if count >= s.config.Expect {
-				// we've met expected limit - add servers
-				s.config.RaftConfig.EnableSingleNode = false
-				for _, member := range members {
-					valid, p := isConsulServer(member)
-					if valid && member.Name != m.Name && p.Expect != parts.Expect {
-						addAddr := &net.TCPAddr{IP: member.Addr, Port: p.Port}
-						future := s.raft.AddPeer(addAddr)
-
-						if err := future.Error(); err != nil && err != raft.ErrKnownPeer {
-							s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err)
-							// hmm....
-						}
-					}
-				}
-			} else {
-				// not enough servers yet
-				return nil
-			}
-		}
-	}
-
 	// Attempt to add as a peer
 	var addr net.Addr = &net.TCPAddr{IP: m.Addr, Port: parts.Port}
 	future := s.raft.AddPeer(addr)

consul/serf.go

@@ -1,8 +1,10 @@
 package consul
 
 import (
-	"github.com/hashicorp/serf/serf"
+	"net"
 	"strings"
+
+	"github.com/hashicorp/serf/serf"
 )
 
 const (
@@ -149,6 +151,41 @@ func (s *Server) nodeJoin(me serf.MemberEvent, wan bool) {
 			s.localConsuls[parts.Addr.String()] = parts
 			s.localLock.Unlock()
 		}
+
+		// If we're still expecting, and they are too, check servers.
+		if s.config.Expect != 0 && parts.Expect != 0 {
+			index, err := s.raftStore.LastIndex()
+			if err == nil && index == 0 {
+				members := s.serfLAN.Members()
+				addrs := make([]net.Addr, 0)
+				for _, member := range members {
+					valid, p := isConsulServer(member)
+					if valid {
+						if p.Expect != parts.Expect {
+							s.logger.Printf("[ERR] consul: '%v' and '%v' have different expect values. All expect nodes should have the same value, will never leave expect mode", m.Name, member.Name)
+							return
+						} else {
+							addrs = append(addrs, &net.TCPAddr{IP: member.Addr, Port: p.Port})
+						}
+					}
+				}
+
+				if len(addrs) >= s.config.Expect {
+					// we have enough nodes, set peers.
+
+					future := s.raft.SetPeers(addrs)
+
+					if err := future.Error(); err != nil {
+						s.logger.Printf("[ERR] consul: failed to leave expect mode and set peers: %v", err)
+					} else {
+						// we've left expect mode, don't enter this again
+						s.config.Expect = 0
+					}
+				}
+			} else if err != nil {
+				s.logger.Printf("[ERR] consul: error retrieving index: %v", err)
+			}
+		}
 	}
 }
 

consul/server.go

@@ -254,8 +254,8 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, w
 
 // setupRaft is used to setup and initialize Raft
 func (s *Server) setupRaft() error {
-	// If we are in bootstrap or expect mode, enable a single node cluster
-	if s.config.Bootstrap || s.config.Expect != 0 {
+	// If we are in bootstrap mode, enable a single node cluster
+	if s.config.Bootstrap {
 		s.config.RaftConfig.EnableSingleNode = true
 	}