2019-09-10 02:15:52 +00:00
|
|
|
## Nim-LibP2P
|
2019-09-24 17:48:23 +00:00
|
|
|
## Copyright (c) 2019 Status Research & Development GmbH
|
2019-09-10 02:15:52 +00:00
|
|
|
## Licensed under either of
|
|
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
|
|
## at your option.
|
|
|
|
## This file may not be copied, modified, or distributed except according to
|
|
|
|
## those terms.
|
|
|
|
|
2020-09-24 16:43:20 +00:00
|
|
|
import std/[sequtils, strutils, tables, hashes]
|
2020-06-11 18:09:34 +00:00
|
|
|
import chronos, chronicles, nimcrypto/sha2, metrics
|
2019-12-10 20:50:35 +00:00
|
|
|
import rpc/[messages, message, protobuf],
|
2020-07-01 06:25:09 +00:00
|
|
|
../../peerid,
|
2019-09-24 16:16:39 +00:00
|
|
|
../../peerinfo,
|
2020-06-19 17:29:43 +00:00
|
|
|
../../stream/connection,
|
2019-09-12 05:46:08 +00:00
|
|
|
../../crypto/crypto,
|
2020-03-23 06:03:36 +00:00
|
|
|
../../protobuf/minprotobuf,
|
|
|
|
../../utility
|
2019-09-10 02:15:52 +00:00
|
|
|
|
2020-09-24 16:43:20 +00:00
|
|
|
export peerid, connection
|
|
|
|
|
2019-09-10 02:15:52 +00:00
|
|
|
logScope:
|
2020-12-01 17:34:27 +00:00
|
|
|
topics = "libp2p pubsubpeer"
|
2019-09-10 02:15:52 +00:00
|
|
|
|
2020-08-04 23:27:59 +00:00
|
|
|
when defined(libp2p_expensive_metrics):
|
|
|
|
declareCounter(libp2p_pubsub_sent_messages, "number of messages sent", labels = ["id", "topic"])
|
|
|
|
declareCounter(libp2p_pubsub_received_messages, "number of messages received", labels = ["id", "topic"])
|
|
|
|
declareCounter(libp2p_pubsub_skipped_received_messages, "number of received skipped messages", labels = ["id"])
|
|
|
|
declareCounter(libp2p_pubsub_skipped_sent_messages, "number of sent skipped messages", labels = ["id"])
|
2020-06-12 02:20:58 +00:00
|
|
|
|
2019-09-10 02:15:52 +00:00
|
|
|
type
|
2020-04-30 13:22:31 +00:00
|
|
|
PubSubObserver* = ref object
|
2020-05-29 16:46:27 +00:00
|
|
|
onRecv*: proc(peer: PubSubPeer; msgs: var RPCMsg) {.gcsafe, raises: [Defect].}
|
|
|
|
onSend*: proc(peer: PubSubPeer; msgs: var RPCMsg) {.gcsafe, raises: [Defect].}
|
2019-09-10 02:15:52 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
PubSubPeerEventKind* {.pure.} = enum
|
|
|
|
Connected
|
|
|
|
Disconnected
|
|
|
|
|
|
|
|
PubsubPeerEvent* = object
|
|
|
|
kind*: PubSubPeerEventKind
|
|
|
|
|
|
|
|
GetConn* = proc(): Future[Connection] {.gcsafe.}
|
|
|
|
OnEvent* = proc(peer: PubSubPeer, event: PubsubPeerEvent) {.gcsafe.}
|
2020-09-01 07:33:03 +00:00
|
|
|
|
2020-04-30 13:22:31 +00:00
|
|
|
PubSubPeer* = ref object of RootObj
|
2020-09-01 07:33:03 +00:00
|
|
|
getConn*: GetConn # callback to establish a new send connection
|
2020-09-22 07:05:53 +00:00
|
|
|
onEvent*: OnEvent # Connectivity updates for peer
|
2020-08-12 00:05:49 +00:00
|
|
|
codec*: string # the protocol that this peer joined from
|
2020-09-22 07:05:53 +00:00
|
|
|
sendConn*: Connection # cached send connection
|
2020-09-01 07:33:03 +00:00
|
|
|
connections*: seq[Connection] # connections to this peer
|
2020-08-12 00:05:49 +00:00
|
|
|
peerId*: PeerID
|
2020-04-30 13:22:31 +00:00
|
|
|
handler*: RPCHandler
|
|
|
|
observers*: ref seq[PubSubObserver] # ref as in smart_ptr
|
|
|
|
|
2020-09-21 09:16:29 +00:00
|
|
|
score*: float64
|
|
|
|
iWantBudget*: int
|
|
|
|
iHaveBudget*: int
|
|
|
|
outbound*: bool # if this is an outbound connection
|
|
|
|
appScore*: float64 # application specific score
|
|
|
|
behaviourPenalty*: float64 # the eventual penalty score
|
2021-01-08 05:21:24 +00:00
|
|
|
|
|
|
|
when defined(libp2p_agents_metrics):
|
|
|
|
shortAgent*: string
|
2020-09-21 09:16:29 +00:00
|
|
|
|
2020-09-01 07:33:03 +00:00
|
|
|
RPCHandler* = proc(peer: PubSubPeer, msg: RPCMsg): Future[void] {.gcsafe.}
|
2019-09-10 02:15:52 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
func hash*(p: PubSubPeer): Hash =
|
2020-07-13 13:32:38 +00:00
|
|
|
# int is either 32/64, so intptr basically, pubsubpeer is a ref
|
|
|
|
cast[pointer](p).hash
|
|
|
|
|
2020-09-06 08:31:47 +00:00
|
|
|
func shortLog*(p: PubSubPeer): string =
|
|
|
|
if p.isNil: "PubSubPeer(nil)"
|
|
|
|
else: shortLog(p.peerId)
|
|
|
|
chronicles.formatIt(PubSubPeer): shortLog(it)
|
2019-12-06 02:16:18 +00:00
|
|
|
|
2020-06-29 15:15:31 +00:00
|
|
|
proc connected*(p: PubSubPeer): bool =
|
2020-08-12 00:05:49 +00:00
|
|
|
not p.sendConn.isNil and not
|
|
|
|
(p.sendConn.closed or p.sendConn.atEof)
|
2020-07-08 00:33:05 +00:00
|
|
|
|
2020-09-24 16:43:20 +00:00
|
|
|
proc hasObservers(p: PubSubPeer): bool =
|
|
|
|
p.observers != nil and anyIt(p.observers[], it != nil)
|
|
|
|
|
2020-05-29 16:46:27 +00:00
|
|
|
proc recvObservers(p: PubSubPeer, msg: var RPCMsg) =
|
|
|
|
# trigger hooks
|
|
|
|
if not(isNil(p.observers)) and p.observers[].len > 0:
|
|
|
|
for obs in p.observers[]:
|
2020-06-24 15:08:44 +00:00
|
|
|
if not(isNil(obs)): # TODO: should never be nil, but...
|
|
|
|
obs.onRecv(p, msg)
|
2020-05-29 16:46:27 +00:00
|
|
|
|
|
|
|
proc sendObservers(p: PubSubPeer, msg: var RPCMsg) =
|
|
|
|
# trigger hooks
|
|
|
|
if not(isNil(p.observers)) and p.observers[].len > 0:
|
|
|
|
for obs in p.observers[]:
|
2020-06-24 15:08:44 +00:00
|
|
|
if not(isNil(obs)): # TODO: should never be nil, but...
|
|
|
|
obs.onSend(p, msg)
|
2020-05-29 16:46:27 +00:00
|
|
|
|
2019-12-17 05:24:03 +00:00
|
|
|
proc handle*(p: PubSubPeer, conn: Connection) {.async.} =
|
2020-09-06 08:31:47 +00:00
|
|
|
debug "starting pubsub read loop",
|
|
|
|
conn, peer = p, closed = conn.closed
|
2019-09-10 02:15:52 +00:00
|
|
|
try:
|
2020-05-25 14:33:24 +00:00
|
|
|
try:
|
2020-08-03 05:20:11 +00:00
|
|
|
while not conn.atEof:
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "waiting for data", conn, peer = p, closed = conn.closed
|
|
|
|
|
2020-08-12 00:05:49 +00:00
|
|
|
let data = await conn.readLp(64 * 1024)
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "read data from peer",
|
|
|
|
conn, peer = p, closed = conn.closed,
|
|
|
|
data = data.shortLog
|
2020-05-25 14:33:24 +00:00
|
|
|
|
2020-07-15 08:25:39 +00:00
|
|
|
var rmsg = decodeRpcMsg(data)
|
|
|
|
if rmsg.isErr():
|
2020-09-06 08:31:47 +00:00
|
|
|
notice "failed to decode msg from peer",
|
|
|
|
conn, peer = p, closed = conn.closed,
|
|
|
|
err = rmsg.error()
|
2020-07-15 08:25:39 +00:00
|
|
|
break
|
|
|
|
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "decoded msg from peer",
|
|
|
|
conn, peer = p, closed = conn.closed,
|
|
|
|
msg = rmsg.get().shortLog
|
2020-05-25 14:33:24 +00:00
|
|
|
# trigger hooks
|
2020-09-04 06:10:32 +00:00
|
|
|
p.recvObservers(rmsg.get())
|
2020-06-02 23:53:38 +00:00
|
|
|
|
2020-08-04 23:27:59 +00:00
|
|
|
when defined(libp2p_expensive_metrics):
|
2020-09-04 06:10:32 +00:00
|
|
|
for m in rmsg.get().messages:
|
2020-08-04 23:27:59 +00:00
|
|
|
for t in m.topicIDs:
|
|
|
|
# metrics
|
2020-09-06 08:31:47 +00:00
|
|
|
libp2p_pubsub_received_messages.inc(labelValues = [$p.peerId, t])
|
2020-06-11 18:09:34 +00:00
|
|
|
|
2020-09-04 06:10:32 +00:00
|
|
|
await p.handler(p, rmsg.get())
|
2020-05-25 14:33:24 +00:00
|
|
|
finally:
|
|
|
|
await conn.close()
|
2020-09-04 16:30:45 +00:00
|
|
|
except CancelledError:
|
|
|
|
# This is top-level procedure which will work as separate task, so it
|
2020-11-23 21:02:23 +00:00
|
|
|
# do not need to propagate CancelledError.
|
2020-09-04 16:30:45 +00:00
|
|
|
trace "Unexpected cancellation in PubSubPeer.handle"
|
2019-12-06 02:16:18 +00:00
|
|
|
except CatchableError as exc:
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "Exception occurred in PubSubPeer.handle",
|
|
|
|
conn, peer = p, closed = conn.closed, exc = exc.msg
|
2020-09-01 07:33:03 +00:00
|
|
|
finally:
|
2020-09-06 08:31:47 +00:00
|
|
|
debug "exiting pubsub read loop",
|
|
|
|
conn, peer = p, closed = conn.closed
|
2019-09-10 02:15:52 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
proc connectOnce(p: PubSubPeer): Future[void] {.async.} =
|
2020-08-17 19:39:39 +00:00
|
|
|
try:
|
2020-09-22 07:05:53 +00:00
|
|
|
let newConn = await p.getConn()
|
2020-08-17 19:39:39 +00:00
|
|
|
if newConn.isNil:
|
2020-09-22 07:05:53 +00:00
|
|
|
raise (ref CatchableError)(msg: "Cannot establish send connection")
|
2020-08-17 19:39:39 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
# When the send channel goes up, subscriptions need to be sent to the
|
|
|
|
# remote peer - if we had multiple channels up and one goes down, all
|
|
|
|
# stop working so we make an effort to only keep a single channel alive
|
2020-09-01 07:33:03 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
trace "Get new send connection", p, newConn
|
2020-08-17 19:39:39 +00:00
|
|
|
p.sendConn = newConn
|
2020-09-22 07:05:53 +00:00
|
|
|
|
|
|
|
if p.onEvent != nil:
|
|
|
|
p.onEvent(p, PubsubPeerEvent(kind: PubSubPeerEventKind.Connected))
|
|
|
|
|
|
|
|
await handle(p, newConn)
|
2020-08-17 19:39:39 +00:00
|
|
|
finally:
|
2020-09-22 07:05:53 +00:00
|
|
|
if p.sendConn != nil:
|
|
|
|
trace "Removing send connection", p, conn = p.sendConn
|
|
|
|
await p.sendConn.close()
|
remove send lock (#334)
* remove send lock
When mplex receives data it will block until a reader has processed the
data. Thus, when a large message is received, such as a gossipsub
subscription table, all of mplex will be blocked until all reading is
finished.
However, if at the same time a `dial` to establish a gossipsub send
connection is ongoing, that `dial` will be blocked because mplex is no
longer reading data - specifically, it might indeed be the connection
that's processing the previous data that is waiting for a send
connection.
There are other problems with the current code:
* If an exception is raised, it is not necessarily raised for the same
connection as `p.sendConn`, so resetting `p.sendConn` in the exception
handling is wrong
* `p.isConnected` is checked before taking the lock - thus, if it
returns false, a new dial will be started. If a new task enters `send`
before dial is finished, it will also determine `p.isConnected` is
false, then get stuck on the lock - when the previous task finishes and
releases the lock, the new task will _also_ dial and thus reset
`p.sendConn` causing a leak.
* prefer existing connection
simplifies flow
2020-08-17 10:38:27 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
p.sendConn = nil
|
|
|
|
if p.onEvent != nil:
|
|
|
|
p.onEvent(p, PubsubPeerEvent(kind: PubSubPeerEventKind.Disconnected))
|
|
|
|
|
|
|
|
proc connectImpl(p: PubSubPeer) {.async.} =
|
2020-09-01 07:33:03 +00:00
|
|
|
try:
|
2020-09-22 07:05:53 +00:00
|
|
|
# Keep trying to establish a connection while it's possible to do so - the
|
|
|
|
# send connection might get disconnected due to a timeout or an unrelated
|
|
|
|
# issue so we try to get a new on
|
|
|
|
while true:
|
|
|
|
await connectOnce(p)
|
|
|
|
|
2020-09-01 07:33:03 +00:00
|
|
|
except CatchableError as exc:
|
2020-09-22 07:05:53 +00:00
|
|
|
debug "Could not establish send connection", msg = exc.msg
|
2020-09-01 07:33:03 +00:00
|
|
|
|
|
|
|
proc connect*(p: PubSubPeer) =
|
2020-09-22 07:05:53 +00:00
|
|
|
asyncSpawn connectImpl(p)
|
2020-08-12 00:05:49 +00:00
|
|
|
|
2020-09-24 16:43:20 +00:00
|
|
|
proc sendImpl(conn: Connection, encoded: seq[byte]) {.async.} =
|
2020-08-12 00:05:49 +00:00
|
|
|
try:
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "sending encoded msgs to peer", conn, encoded = shortLog(encoded)
|
2020-09-01 07:33:03 +00:00
|
|
|
await conn.writeLp(encoded)
|
2020-09-06 08:31:47 +00:00
|
|
|
trace "sent pubsub message to remote", conn
|
2020-08-03 05:20:11 +00:00
|
|
|
|
2020-07-09 17:16:46 +00:00
|
|
|
except CatchableError as exc:
|
2020-09-01 07:33:03 +00:00
|
|
|
# Because we detach the send call from the currently executing task using
|
2020-09-22 07:05:53 +00:00
|
|
|
# asyncSpawn, no exceptions may leak out of it
|
2020-09-24 16:43:20 +00:00
|
|
|
trace "Unable to send to remote", conn, msg = exc.msg
|
remove send lock (#334)
* remove send lock
When mplex receives data it will block until a reader has processed the
data. Thus, when a large message is received, such as a gossipsub
subscription table, all of mplex will be blocked until all reading is
finished.
However, if at the same time a `dial` to establish a gossipsub send
connection is ongoing, that `dial` will be blocked because mplex is no
longer reading data - specifically, it might indeed be the connection
that's processing the previous data that is waiting for a send
connection.
There are other problems with the current code:
* If an exception is raised, it is not necessarily raised for the same
connection as `p.sendConn`, so resetting `p.sendConn` in the exception
handling is wrong
* `p.isConnected` is checked before taking the lock - thus, if it
returns false, a new dial will be started. If a new task enters `send`
before dial is finished, it will also determine `p.isConnected` is
false, then get stuck on the lock - when the previous task finishes and
releases the lock, the new task will _also_ dial and thus reset
`p.sendConn` causing a leak.
* prefer existing connection
simplifies flow
2020-08-17 10:38:27 +00:00
|
|
|
# Next time sendConn is used, it will be have its close flag set and thus
|
|
|
|
# will be recycled
|
2020-07-16 10:06:57 +00:00
|
|
|
|
2020-09-22 07:05:53 +00:00
|
|
|
await conn.close() # This will clean up the send connection
|
2020-09-01 07:33:03 +00:00
|
|
|
|
2020-11-23 21:02:23 +00:00
|
|
|
template sendMetrics(msg: RPCMsg): untyped =
|
|
|
|
when defined(libp2p_expensive_metrics):
|
|
|
|
for x in msg.messages:
|
|
|
|
for t in x.topicIDs:
|
|
|
|
# metrics
|
|
|
|
libp2p_pubsub_sent_messages.inc(labelValues = [$p.peerId, t])
|
|
|
|
|
2020-09-25 16:39:34 +00:00
|
|
|
proc send*(p: PubSubPeer, msg: RPCMsg, anonymize: bool) =
|
2020-09-24 16:43:20 +00:00
|
|
|
doAssert(not isNil(p), "pubsubpeer nil!")
|
|
|
|
|
|
|
|
let conn = p.sendConn
|
|
|
|
if conn == nil or conn.closed():
|
|
|
|
trace "No send connection, skipping message", p, msg
|
|
|
|
return
|
|
|
|
|
|
|
|
trace "sending msg to peer", peer = p, rpcMsg = shortLog(msg)
|
|
|
|
|
2020-09-25 16:39:34 +00:00
|
|
|
# When sending messages, we take care to re-encode them with the right
|
|
|
|
# anonymization flag to ensure that we're not penalized for sending invalid
|
|
|
|
# or malicious data on the wire - in particular, re-encoding protects against
|
|
|
|
# some forms of valid but redundantly encoded protobufs with unknown or
|
|
|
|
# duplicated fields
|
2020-09-24 16:43:20 +00:00
|
|
|
let encoded = if p.hasObservers():
|
2020-11-23 21:02:23 +00:00
|
|
|
var mm = msg
|
2020-09-24 16:43:20 +00:00
|
|
|
# trigger send hooks
|
|
|
|
p.sendObservers(mm)
|
2020-11-23 21:02:23 +00:00
|
|
|
sendMetrics(mm)
|
2020-09-25 16:39:34 +00:00
|
|
|
encodeRpcMsg(mm, anonymize)
|
2020-09-24 16:43:20 +00:00
|
|
|
else:
|
|
|
|
# If there are no send hooks, we redundantly re-encode the message to
|
|
|
|
# protobuf for every peer - this could easily be improved!
|
2020-11-23 21:02:23 +00:00
|
|
|
sendMetrics(msg)
|
2020-09-25 16:39:34 +00:00
|
|
|
encodeRpcMsg(msg, anonymize)
|
2020-09-24 16:43:20 +00:00
|
|
|
|
|
|
|
if encoded.len <= 0:
|
|
|
|
debug "empty message, skipping", p, msg
|
|
|
|
return
|
|
|
|
|
|
|
|
# To limit the size of the closure, we only pass the encoded message and
|
|
|
|
# connection to the spawned send task
|
|
|
|
asyncSpawn sendImpl(conn, encoded)
|
2019-12-06 02:16:18 +00:00
|
|
|
|
2020-08-12 00:05:49 +00:00
|
|
|
proc newPubSubPeer*(peerId: PeerID,
|
2020-09-01 07:33:03 +00:00
|
|
|
getConn: GetConn,
|
2020-09-22 07:05:53 +00:00
|
|
|
onEvent: OnEvent,
|
2020-08-12 00:05:49 +00:00
|
|
|
codec: string): PubSubPeer =
|
2020-09-22 07:05:53 +00:00
|
|
|
PubSubPeer(
|
|
|
|
getConn: getConn,
|
|
|
|
onEvent: onEvent,
|
|
|
|
codec: codec,
|
|
|
|
peerId: peerId,
|
|
|
|
)
|