From 5f2e1c9b072c543a906cbea5fcf522ec5bc60a8c Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Wed, 6 Sep 2017 13:53:01 -0500 Subject: [PATCH 1/3] Change member join reconcile step to process joining itself, to handle node IP address changes correctly when number of servers < 3 --- agent/consul/leader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 31d197b675..7768842b8c 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -626,8 +626,8 @@ func (s *Server) handleDeregisterMember(reason string, member serf.Member) error // joinConsulServer is used to try to join another consul server func (s *Server) joinConsulServer(m serf.Member, parts *metadata.Server) error { - // Do not join ourself - if m.Name == s.config.NodeName { + // Do not join ourself if we are the only member + if m.Name == s.config.NodeName && len(s.serfLAN.Members()) == 1 { return nil } From 353565259512bcf6e62ef1a869495d411ef3793c Mon Sep 17 00:00:00 2001 From: James Phillips Date: Wed, 6 Sep 2017 13:05:51 -0700 Subject: [PATCH 2/3] Uses the Raft configuration for the self-add skip check. --- agent/consul/leader.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 7768842b8c..5c1a7ea4a1 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -626,11 +626,6 @@ func (s *Server) handleDeregisterMember(reason string, member serf.Member) error // joinConsulServer is used to try to join another consul server func (s *Server) joinConsulServer(m serf.Member, parts *metadata.Server) error { - // Do not join ourself if we are the only member - if m.Name == s.config.NodeName && len(s.serfLAN.Members()) == 1 { - return nil - } - // Check for possibility of multiple bootstrap nodes if parts.Bootstrap { members := s.serfLAN.Members() @@ -650,15 +645,25 @@ func (s *Server) joinConsulServer(m serf.Member, parts *metadata.Server) error { return err } - // See if it's already in the configuration. It's harmless to re-add it - // but we want to avoid doing that if possible to prevent useless Raft - // log entries. If the address is the same but the ID changed, remove the - // old server before adding the new one. + // Processing ourselves could result in trying to remove ourselves to + // fix up our address, which would make us step down. This is only + // safe to attempt if there are multiple servers available. configFuture := s.raft.GetConfiguration() if err := configFuture.Error(); err != nil { s.logger.Printf("[ERR] consul: failed to get raft configuration: %v", err) return err } + if m.Name == s.config.NodeName { + if l := len(configFuture.Configuration().Servers); l < 3 { + s.logger.Printf("[DEBUG] consul: Skipping self join check for %q since the cluster is too small", m.Name) + return nil + } + } + + // See if it's already in the configuration. It's harmless to re-add it + // but we want to avoid doing that if possible to prevent useless Raft + // log entries. If the address is the same but the ID changed, remove the + // old server before adding the new one. for _, server := range configFuture.Configuration().Servers { // No-op if the raft version is too low if server.Address == raft.ServerAddress(addr) && (minRaftProtocol < 2 || parts.RaftVersion < 3) { From 084679ab65394cb31bfe70d6179035e06128cec7 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Wed, 6 Sep 2017 13:07:42 -0700 Subject: [PATCH 3/3] Pulls down some code for the check loop. --- agent/consul/leader.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 5c1a7ea4a1..2ad4aea820 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -638,13 +638,6 @@ func (s *Server) joinConsulServer(m serf.Member, parts *metadata.Server) error { } } - addr := (&net.TCPAddr{IP: m.Addr, Port: parts.Port}).String() - - minRaftProtocol, err := ServerMinRaftProtocol(s.serfLAN.Members()) - if err != nil { - return err - } - // Processing ourselves could result in trying to remove ourselves to // fix up our address, which would make us step down. This is only // safe to attempt if there are multiple servers available. @@ -664,6 +657,11 @@ func (s *Server) joinConsulServer(m serf.Member, parts *metadata.Server) error { // but we want to avoid doing that if possible to prevent useless Raft // log entries. If the address is the same but the ID changed, remove the // old server before adding the new one. + addr := (&net.TCPAddr{IP: m.Addr, Port: parts.Port}).String() + minRaftProtocol, err := ServerMinRaftProtocol(s.serfLAN.Members()) + if err != nil { + return err + } for _, server := range configFuture.Configuration().Servers { // No-op if the raft version is too low if server.Address == raft.ServerAddress(addr) && (minRaftProtocol < 2 || parts.RaftVersion < 3) {