Preventing multiple nodes in bootstrap mode from adding each other as Raft peers

consul/leader.go

@@ -150,7 +150,7 @@ func (s *Server) handleAliveMember(member serf.Member) error {
 		}
 
 		// Attempt to join the consul server
-		if err := s.joinConsulServer(member, parts.Port); err != nil {
+		if err := s.joinConsulServer(member, parts); err != nil {
 			return err
 		}
 	}
@@ -263,14 +263,26 @@ func (s *Server) handleLeftMember(member serf.Member) error {
 }
 
 // joinConsulServer is used to try to join another consul server
-func (s *Server) joinConsulServer(m serf.Member, port int) error {
+func (s *Server) joinConsulServer(m serf.Member, parts *serverParts) error {
 	// Do not join ourself
 	if m.Name == s.config.NodeName {
 		return nil
 	}
 
+	// Check for possibility of multiple bootstrap nodes
+	if parts.HasFlag(bootstrapFlag) {
+		members := s.serfLAN.Members()
+		for _, member := range members {
+			valid, p := isConsulServer(member)
+			if valid && member.Name != m.Name && p.HasFlag(bootstrapFlag) {
+				s.logger.Printf("[ERR] consul: '%v' and '%v' are both in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.", m.Name, member.Name)
+				return nil
+			}
+		}
+	}
+
 	// Attempt to add as a peer
-	var addr net.Addr = &net.TCPAddr{IP: m.Addr, Port: port}
+	var addr net.Addr = &net.TCPAddr{IP: m.Addr, Port: parts.Port}
 	future := s.raft.AddPeer(addr)
 	if err := future.Error(); err != nil && err != raft.KnownPeer {
 		s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err)

consul/leader_test.go

@@ -257,3 +257,48 @@ CHECK2:
 		}
 	}
 }
+
+func TestLeader_MultiBootstrap(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	dir2, s2 := testServer(t)
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	servers := []*Server{s1, s2}
+
+	// Try to join
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Wait until we have 2 peers
+	start := time.Now()
+CHECK1:
+	for _, s := range servers {
+		peers := s.serfLAN.Members()
+		if len(peers) != 2 {
+			if time.Now().Sub(start) >= 2*time.Second {
+				t.Fatalf("should have 2 peers")
+			} else {
+				time.Sleep(100 * time.Millisecond)
+				goto CHECK1
+			}
+		}
+	}
+
+	// Wait to ensure no peer is added
+	time.Sleep(200 * time.Millisecond)
+
+	// Ensure we don't have multiple raft peers
+	for _, s := range servers {
+		peers, _ := s.raftPeers.Peers()
+		if len(peers) != 1 {
+			t.Fatalf("should only have 1 raft peer!")
+		}
+	}
+}

consul/server.go

@@ -17,6 +17,7 @@ const (
 	serfLANSnapshot = "serf/local.snapshot"
 	serfWANSnapshot = "serf/remote.snapshot"
 	raftState       = "raft/"
+	bootstrapFlag   = "b"
 )
 
 // Server is Consul server which manages the service discovery,
@@ -163,7 +164,7 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (
 	addr := s.rpcListener.Addr().(*net.TCPAddr)
 	flags := ""
 	if s.config.Bootstrap {
-		flags = "b"
+		flags = bootstrapFlag
 	}
 	conf.NodeName = s.config.NodeName
 	conf.Role = fmt.Sprintf("consul:%s:%d:%s", s.config.Datacenter, addr.Port, flags)

consul/util.go

@@ -26,6 +26,11 @@ type serverParts struct {
 	Flags      string
 }
 
+// HasFlag is used to check if a flag is present
+func (s *serverParts) HasFlag(flag string) bool {
+	return strings.Contains(s.Flags, flag)
+}
+
 func init() {
 	// Add each private block
 	privateBlocks = make([]*net.IPNet, 3)