add more heartbeat locking to prevent races

libp2p/protocols/pubsub/gossipsub.nim

@@ -220,10 +220,12 @@ method handleDisconnect*(g: GossipSub, peer: PubSubPeer) {.async.} =
   ## handle peer disconnects
   trace "peer disconnected", peer=peer.id
 
+  # must avoid running this while manipulating mesh/gossip tables
+  await g.heartbeatLock.acquire()
+  try:
     await procCall FloodSub(g).handleDisconnect(peer)
 
     for t in toSeq(g.gossipsub.keys):
-    if t in g.gossipsub:
       g.gossipsub[t].excl(peer.id)
 
       libp2p_gossipsub_peers_per_topic_gossipsub
@@ -233,18 +235,18 @@ method handleDisconnect*(g: GossipSub, peer: PubSubPeer) {.async.} =
       await procCall PubSub(g).subscribeTopic(t, false, peer.id)
 
     for t in toSeq(g.mesh.keys):
-    if t in g.mesh:
       g.mesh[t].excl(peer.id)
 
       libp2p_gossipsub_peers_per_topic_mesh
         .set(g.mesh[t].len.int64, labelValues = [t])
 
     for t in toSeq(g.fanout.keys):
-    if t in g.fanout:
       g.fanout[t].excl(peer.id)
 
       libp2p_gossipsub_peers_per_topic_fanout
         .set(g.fanout[t].len.int64, labelValues = [t])
+  finally:
+    g.heartbeatLock.release()
 
 method subscribeToPeer*(p: GossipSub,
                         conn: Connection) {.async.} =
@@ -255,6 +257,10 @@ method subscribeTopic*(g: GossipSub,
                        topic: string,
                        subscribe: bool,
                        peerId: string) {.gcsafe, async.} =
+  
+  # must avoid running this while manipulating mesh/gossip tables
+  await g.heartbeatLock.acquire()
+  try:
     await procCall PubSub(g).subscribeTopic(topic, subscribe, peerId)
 
     if topic notin g.gossipsub:
@@ -273,10 +279,8 @@ method subscribeTopic*(g: GossipSub,
       .set(g.gossipsub[topic].len.int64, labelValues = [topic])
 
     trace "gossip peers", peers = g.gossipsub[topic].len, topic
-
-  # also rebalance current topic if we are subbed to
-  if topic in g.topics:
-    await g.rebalanceMesh(topic)
+  finally:
+    g.heartbeatLock.release()
 
 proc handleGraft(g: GossipSub,
                  peer: PubSubPeer,

libp2p/protocols/pubsub/pubsub.nim

@@ -100,7 +100,7 @@ method rpcHandler*(p: PubSub,
 method handleDisconnect*(p: PubSub, peer: PubSubPeer) {.async, base.} =
   ## handle peer disconnects
   if peer.id in p.peers:
-    trace "deleting peer", id = peer.id
+    trace "deleting peer", id = peer.id, trace = getStackTrace()
     p.peers.del(peer.id)
 
   # metrics