diff --git a/libp2p/protocols/pubsub/gossipsub.nim b/libp2p/protocols/pubsub/gossipsub.nim
index beee0be..6fd3a8c 100644
--- a/libp2p/protocols/pubsub/gossipsub.nim
+++ b/libp2p/protocols/pubsub/gossipsub.nim
@@ -198,7 +198,7 @@ method unsubscribePeer*(g: GossipSub, peer: PeerID) =
   # remove from peer IPs collection too
   if pubSubPeer.sendConn != nil:
     g.peersInIP.withValue(pubSubPeer.sendConn.observedAddr, s):
-      s[].excl(pubSubPeer)
+      s[].excl(pubSubPeer.peerId)
       if s[].len == 0:
         g.peersInIP.del(pubSubPeer.sendConn.observedAddr)
 
diff --git a/libp2p/protocols/pubsub/gossipsub/scoring.nim b/libp2p/protocols/pubsub/gossipsub/scoring.nim
index fa8daae..494ad21 100644
--- a/libp2p/protocols/pubsub/gossipsub/scoring.nim
+++ b/libp2p/protocols/pubsub/gossipsub/scoring.nim
@@ -46,7 +46,7 @@ proc colocationFactor(g: GossipSub, peer: PubSubPeer): float64 =
     let
       address = peer.sendConn.observedAddr
 
-    g.peersInIP.mgetOrPut(address, initHashSet[PubSubPeer]()).incl(peer)
+    g.peersInIP.mgetOrPut(address, initHashSet[PeerID]()).incl(peer.peerId)
     let
       ipPeers = g.peersInIP[address]
       len = ipPeers.len.float64
diff --git a/libp2p/protocols/pubsub/gossipsub/types.nim b/libp2p/protocols/pubsub/gossipsub/types.nim
index 59844cc..c9fced1 100644
--- a/libp2p/protocols/pubsub/gossipsub/types.nim
+++ b/libp2p/protocols/pubsub/gossipsub/types.nim
@@ -146,7 +146,7 @@ type
     parameters*: GossipSubParams
     topicParams*: Table[string, TopicParams]
     directPeersLoop*: Future[void]
-    peersInIP*: Table[MultiAddress, HashSet[PubSubPeer]]
+    peersInIP*: Table[MultiAddress, HashSet[PeerID]]
 
     heartbeatEvents*: seq[AsyncEvent]