diff --git a/go.mod b/go.mod index 45e6f99227..5fd5550b62 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5 github.com/hashicorp/logutils v1.0.0 github.com/hashicorp/mdns v1.0.1 // indirect - github.com/hashicorp/memberlist v0.1.4 + github.com/hashicorp/memberlist v0.1.5 github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/raft v1.1.1 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea diff --git a/go.sum b/go.sum index d40096b9ac..7da244e07b 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/hashicorp/mdns v1.0.1 h1:XFSOubp8KWB+Jd2PDyaX5xUd5bhSP/+pTDZVDMzZJM8= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs= -github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= +github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= github.com/hashicorp/raft v1.1.1 h1:HJr7UE1x/JrJSc9Oy6aDBHtNHUUBHjcQjTgvUVihoZs= diff --git a/vendor/github.com/hashicorp/memberlist/net.go b/vendor/github.com/hashicorp/memberlist/net.go index f6a0d45fed..068d8e1ade 100644 --- a/vendor/github.com/hashicorp/memberlist/net.go +++ b/vendor/github.com/hashicorp/memberlist/net.go @@ -522,7 +522,7 @@ func (m *Memberlist) handleIndirectPing(buf []byte, from net.Addr) { // Send the ping. addr := joinHostPort(net.IP(ind.Target).String(), ind.Port) if err := m.encodeAndSendMsg(addr, pingMsg, &ping); err != nil { - m.logger.Printf("[ERR] memberlist: Failed to send ping: %s %s", err, LogAddress(from)) + m.logger.Printf("[ERR] memberlist: Failed to send indirect ping: %s %s", err, LogAddress(from)) } // Setup a timer to fire off a nack if no ack is seen in time. diff --git a/vendor/github.com/hashicorp/memberlist/state.go b/vendor/github.com/hashicorp/memberlist/state.go index 1af62943e8..f5ed65a782 100644 --- a/vendor/github.com/hashicorp/memberlist/state.go +++ b/vendor/github.com/hashicorp/memberlist/state.go @@ -6,6 +6,7 @@ import ( "math" "math/rand" "net" + "strings" "sync/atomic" "time" @@ -242,6 +243,21 @@ func (m *Memberlist) probeNodeByAddr(addr string) { m.probeNode(n) } +// failedRemote checks the error and decides if it indicates a failure on the +// other end. +func failedRemote(err error) bool { + switch t := err.(type) { + case *net.OpError: + if strings.HasPrefix(t.Net, "tcp") { + switch t.Op { + case "dial", "read", "write": + return true + } + } + } + return false +} + // probeNode handles a single round of failure checking on a node. func (m *Memberlist) probeNode(node *nodeState) { defer metrics.MeasureSince([]string{"memberlist", "probeNode"}, time.Now()) @@ -272,10 +288,20 @@ func (m *Memberlist) probeNode(node *nodeState) { // soon as possible. deadline := sent.Add(probeInterval) addr := node.Address() + + // Arrange for our self-awareness to get updated. + var awarenessDelta int + defer func() { + m.awareness.ApplyDelta(awarenessDelta) + }() if node.State == stateAlive { if err := m.encodeAndSendMsg(addr, pingMsg, &ping); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send ping: %s", err) - return + if failedRemote(err) { + goto HANDLE_REMOTE_FAILURE + } else { + return + } } } else { var msgs [][]byte @@ -296,7 +322,11 @@ func (m *Memberlist) probeNode(node *nodeState) { compound := makeCompoundMessage(msgs) if err := m.rawSendMsgPacket(addr, &node.Node, compound.Bytes()); err != nil { m.logger.Printf("[ERR] memberlist: Failed to send compound ping and suspect message to %s: %s", addr, err) - return + if failedRemote(err) { + goto HANDLE_REMOTE_FAILURE + } else { + return + } } } @@ -305,10 +335,7 @@ func (m *Memberlist) probeNode(node *nodeState) { // which will improve our health until we get to the failure scenarios // at the end of this function, which will alter this delta variable // accordingly. - awarenessDelta := -1 - defer func() { - m.awareness.ApplyDelta(awarenessDelta) - }() + awarenessDelta = -1 // Wait for response or round-trip-time. select { @@ -333,9 +360,10 @@ func (m *Memberlist) probeNode(node *nodeState) { // probe interval it will give the TCP fallback more time, which // is more active in dealing with lost packets, and it gives more // time to wait for indirect acks/nacks. - m.logger.Printf("[DEBUG] memberlist: Failed ping: %v (timeout reached)", node.Name) + m.logger.Printf("[DEBUG] memberlist: Failed ping: %s (timeout reached)", node.Name) } +HANDLE_REMOTE_FAILURE: // Get some random live nodes. m.nodeLock.RLock() kNodes := kRandomNodes(m.config.IndirectChecks, m.nodes, func(n *nodeState) bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index cb9113e450..184260d494 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -264,7 +264,7 @@ github.com/hashicorp/hil/ast github.com/hashicorp/logutils # github.com/hashicorp/mdns v1.0.1 github.com/hashicorp/mdns -# github.com/hashicorp/memberlist v0.1.4 +# github.com/hashicorp/memberlist v0.1.5 github.com/hashicorp/memberlist # github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/net-rpc-msgpackrpc