nim-libp2p/libp2p/multistream.nim

183 lines
5.6 KiB
Nim
Raw Normal View History

2019-08-30 05:16:55 +00:00
## Nim-LibP2P
2019-09-24 17:48:23 +00:00
## Copyright (c) 2019 Status Research & Development GmbH
2019-08-30 05:16:55 +00:00
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
import strutils, stew/byteutils
2019-09-09 17:33:32 +00:00
import chronos, chronicles
2019-10-04 16:04:21 +00:00
import connection,
vbuffer,
errors,
PubSub (Gossip & Flood) Implementation (#36) This adds gossipsub and floodsub, as well as basic interop testing with the go libp2p daemon. * add close event * wip: gossipsub * splitting rpc message * making message handling more consistent * initial gossipsub implementation * feat: nim 1.0 cleanup * wip: gossipsub protobuf * adding encoding/decoding of gossipsub messages * add disconnect handler * add proper gossipsub msg handling * misc: cleanup for nim 1.0 * splitting floodsub and gossipsub tests * feat: add mesh rebalansing * test pubsub * add mesh rebalansing tests * testing mesh maintenance * finishing mcache implementatin * wip: commenting out broken tests * wip: don't run heartbeat for now * switchout debug for trace logging * testing gossip peer selection algorithm * test stream piping * more work around message amplification * get the peerid from message * use timed cache as backing store * allow setting timeout in constructor * several changes to improve performance * more through testing of msg amplification * prevent gc issues * allow piping to self and prevent deadlocks * improove floodsub * allow running hook on cache eviction * prevent race conditions * prevent race conditions and improove tests * use hashes as cache keys * removing useless file * don't create a new seq * re-enable pubsub tests * fix imports * reduce number of runs to speed up tests * break out control message processing * normalize sleeps between steps * implement proper transport filtering * initial interop testing * clean up floodsub publish logic * allow dialing without a protocol * adding multiple reads/writes * use protobuf varint in mplex * don't loose conn's peerInfo * initial interop pubsub tests * don't duplicate connections/peers * bring back interop tests * wip: interop * re-enable interop and daemon tests * add multiple read write tests from handlers * don't cleanup channel prematurely * use correct channel to send/receive msgs * adjust tests with latest changes * include interop tests * remove temp logging output * fix ci * use correct public key serialization * additional tests for pubsub interop
2019-12-06 02:16:18 +00:00
protocols/protocol
2019-08-30 05:16:55 +00:00
2019-09-11 21:06:10 +00:00
logScope:
topic = "Multistream"
2020-01-07 08:02:37 +00:00
const
2019-10-04 16:04:21 +00:00
MsgSize* = 64*1024
Codec* = "/multistream/1.0.0"
MSCodec* = "\x13" & Codec & "\n"
2019-10-04 20:10:01 +00:00
Na* = "\x03na\n"
Ls* = "\x03ls\n"
2019-08-30 05:16:55 +00:00
type
2019-08-30 22:15:47 +00:00
Matcher* = proc (proto: string): bool {.gcsafe.}
2019-08-30 05:16:55 +00:00
HandlerHolder* = object
2019-09-06 00:16:49 +00:00
proto*: string
protocol*: LPProtocol
match*: Matcher
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
MultistreamSelect* = ref object of RootObj
2019-08-30 05:16:55 +00:00
handlers*: seq[HandlerHolder]
codec*: string
2020-02-12 14:40:15 +00:00
MultistreamHandshakeException* = object of CatchableError
proc newMultistreamHandshakeException*(): ref Exception {.inline.} =
result = newException(MultistreamHandshakeException,
"could not perform multistream handshake")
2020-02-04 16:16:21 +00:00
proc newMultistream*(): MultistreamSelect =
2019-08-30 05:16:55 +00:00
new result
2019-09-06 21:25:54 +00:00
result.codec = MSCodec
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect,
2019-08-30 05:16:55 +00:00
conn: Connection,
2019-09-14 13:55:52 +00:00
proto: seq[string]):
Future[string] {.async.} =
trace "initiating handshake", codec = m.codec
2019-08-30 05:16:55 +00:00
## select a remote protocol
2019-09-05 15:19:15 +00:00
await conn.write(m.codec) # write handshake
2019-08-30 05:16:55 +00:00
if proto.len() > 0:
2019-09-14 15:55:58 +00:00
trace "selecting proto", proto = proto
2019-09-05 15:19:15 +00:00
await conn.writeLp((proto[0] & "\n")) # select proto
2019-08-30 05:16:55 +00:00
2020-05-08 20:58:23 +00:00
result = cast[string]((await conn.readLp(1024))) # read ms header
2019-09-06 00:16:49 +00:00
result.removeSuffix("\n")
if result != Codec:
notice "handshake failed", codec = result.toHex()
2020-02-12 14:40:15 +00:00
raise newMultistreamHandshakeException()
2019-08-30 05:16:55 +00:00
if proto.len() == 0: # no protocols, must be a handshake call
2019-09-06 00:16:49 +00:00
return
2019-08-30 05:16:55 +00:00
2020-05-08 20:58:23 +00:00
result = string.fromBytes(await conn.readLp(1024)) # read the first proto
2019-09-14 15:55:58 +00:00
trace "reading first requested proto"
2019-09-06 00:16:49 +00:00
result.removeSuffix("\n")
if result == proto[0]:
trace "succesfully selected ", proto = proto
2019-09-06 00:16:49 +00:00
return
if not result.len > 0:
2019-09-14 15:55:58 +00:00
trace "selecting one of several protos"
for p in proto[1..<proto.len()]:
2019-09-05 15:19:15 +00:00
await conn.writeLp((p & "\n")) # select proto
2020-05-08 20:58:23 +00:00
result = string.fromBytes(await conn.readLp(1024)) # read the first proto
2019-09-06 00:16:49 +00:00
result.removeSuffix("\n")
if result == p:
trace "selected protocol", protocol = result
break
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect,
conn: Connection,
2020-01-07 08:02:37 +00:00
proto: string): Future[bool] {.async.} =
if proto.len > 0:
result = (await m.select(conn, @[proto])) == proto
else:
2019-09-08 07:43:33 +00:00
result = (await m.select(conn, @[])) == Codec
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect, conn: Connection): Future[bool] =
2019-09-08 07:43:33 +00:00
m.select(conn, "")
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
proc list*(m: MultistreamSelect,
2019-08-30 05:16:55 +00:00
conn: Connection): Future[seq[string]] {.async.} =
## list remote protos requests on connection
if not await m.select(conn):
2019-08-30 05:16:55 +00:00
return
await conn.write(Ls) # send ls
2019-08-30 05:16:55 +00:00
var list = newSeq[string]()
2020-05-08 20:58:23 +00:00
let ms = string.fromBytes(await conn.readLp(1024))
2019-08-30 05:16:55 +00:00
for s in ms.split("\n"):
if s.len() > 0:
list.add(s)
result = list
2020-02-04 16:16:21 +00:00
proc handle*(m: MultistreamSelect, conn: Connection) {.async, gcsafe.} =
2019-09-14 15:55:58 +00:00
trace "handle: starting multistream handling"
tryAndWarn "multistream handle":
while not conn.closed:
2020-05-08 20:58:23 +00:00
var ms = string.fromBytes(await conn.readLp(1024))
ms.removeSuffix("\n")
2020-01-07 08:02:37 +00:00
2019-09-14 15:55:58 +00:00
trace "handle: got request for ", ms
if ms.len() <= 0:
2019-09-14 15:55:58 +00:00
trace "handle: invalid proto"
await conn.write(Na)
if m.handlers.len() == 0:
2019-09-14 15:55:58 +00:00
trace "handle: sending `na` for protocol ", protocol = ms
await conn.write(Na)
continue
case ms:
of "ls":
2019-09-14 15:55:58 +00:00
trace "handle: listing protos"
var protos = ""
for h in m.handlers:
protos &= (h.proto & "\n")
await conn.writeLp(protos)
of Codec:
await conn.write(m.codec)
else:
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or ms == h.proto:
2019-09-28 19:54:16 +00:00
trace "found handler for", protocol = ms
await conn.writeLp((h.proto & "\n"))
tryAndWarn "multistream handle handler":
await h.protocol.handler(conn, ms)
2019-09-12 05:45:43 +00:00
return
2019-09-14 13:55:52 +00:00
warn "no handlers for ", protocol = ms
await conn.write(Na)
trace "leaving multistream loop"
# we might be tempted to close conn here but that would be a bad idea!
# we indeed will reuse it later on
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
proc addHandler*[T: LPProtocol](m: MultistreamSelect,
2019-09-03 20:40:19 +00:00
codec: string,
2019-08-30 05:16:55 +00:00
protocol: T,
matcher: Matcher = nil) =
2019-09-14 13:55:52 +00:00
## register a protocol
2020-01-07 08:02:37 +00:00
# TODO: This is a bug in chronicles,
2019-09-14 13:55:52 +00:00
# it break if I uncoment this line.
2020-01-07 08:02:37 +00:00
# Which is almost the same as the
2019-09-14 13:55:52 +00:00
# one on the next override of addHandler
#
2019-09-14 15:55:58 +00:00
# trace "registering protocol", codec = codec
2019-09-14 13:55:52 +00:00
m.handlers.add(HandlerHolder(proto: codec,
protocol: protocol,
match: matcher))
2020-02-04 16:16:21 +00:00
proc addHandler*[T: LPProtoHandler](m: MultistreamSelect,
2019-09-14 13:55:52 +00:00
codec: string,
handler: T,
matcher: Matcher = nil) =
## helper to allow registering pure handlers
2019-09-14 15:55:58 +00:00
trace "registering proto handler", codec = codec
2019-09-14 13:55:52 +00:00
let protocol = new LPProtocol
protocol.codec = codec
protocol.handler = handler
2019-09-03 20:40:19 +00:00
m.handlers.add(HandlerHolder(proto: codec,
2019-08-30 05:16:55 +00:00
protocol: protocol,
match: matcher))