nim-libp2p/libp2p/multistream.nim

206 lines
6.2 KiB
Nim
Raw Normal View History

2019-08-30 05:16:55 +00:00
## Nim-LibP2P
2019-09-24 17:48:23 +00:00
## Copyright (c) 2019 Status Research & Development GmbH
2019-08-30 05:16:55 +00:00
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
2020-06-03 02:21:11 +00:00
import strutils
import chronos, chronicles, stew/byteutils
import stream/connection,
2019-10-04 16:04:21 +00:00
vbuffer,
PubSub (Gossip & Flood) Implementation (#36) This adds gossipsub and floodsub, as well as basic interop testing with the go libp2p daemon. * add close event * wip: gossipsub * splitting rpc message * making message handling more consistent * initial gossipsub implementation * feat: nim 1.0 cleanup * wip: gossipsub protobuf * adding encoding/decoding of gossipsub messages * add disconnect handler * add proper gossipsub msg handling * misc: cleanup for nim 1.0 * splitting floodsub and gossipsub tests * feat: add mesh rebalansing * test pubsub * add mesh rebalansing tests * testing mesh maintenance * finishing mcache implementatin * wip: commenting out broken tests * wip: don't run heartbeat for now * switchout debug for trace logging * testing gossip peer selection algorithm * test stream piping * more work around message amplification * get the peerid from message * use timed cache as backing store * allow setting timeout in constructor * several changes to improve performance * more through testing of msg amplification * prevent gc issues * allow piping to self and prevent deadlocks * improove floodsub * allow running hook on cache eviction * prevent race conditions * prevent race conditions and improove tests * use hashes as cache keys * removing useless file * don't create a new seq * re-enable pubsub tests * fix imports * reduce number of runs to speed up tests * break out control message processing * normalize sleeps between steps * implement proper transport filtering * initial interop testing * clean up floodsub publish logic * allow dialing without a protocol * adding multiple reads/writes * use protobuf varint in mplex * don't loose conn's peerInfo * initial interop pubsub tests * don't duplicate connections/peers * bring back interop tests * wip: interop * re-enable interop and daemon tests * add multiple read write tests from handlers * don't cleanup channel prematurely * use correct channel to send/receive msgs * adjust tests with latest changes * include interop tests * remove temp logging output * fix ci * use correct public key serialization * additional tests for pubsub interop
2019-12-06 02:16:18 +00:00
protocols/protocol
2019-08-30 05:16:55 +00:00
2019-09-11 21:06:10 +00:00
logScope:
topics = "multistream"
2019-09-11 21:06:10 +00:00
2020-01-07 08:02:37 +00:00
const
2019-10-04 16:04:21 +00:00
MsgSize* = 64*1024
Codec* = "/multistream/1.0.0"
MSCodec* = "\x13" & Codec & "\n"
2019-10-04 20:10:01 +00:00
Na* = "\x03na\n"
Ls* = "\x03ls\n"
2019-08-30 05:16:55 +00:00
type
2019-08-30 22:15:47 +00:00
Matcher* = proc (proto: string): bool {.gcsafe.}
2019-08-30 05:16:55 +00:00
HandlerHolder* = object
2019-09-06 00:16:49 +00:00
proto*: string
protocol*: LPProtocol
match*: Matcher
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
MultistreamSelect* = ref object of RootObj
2019-08-30 05:16:55 +00:00
handlers*: seq[HandlerHolder]
codec*: string
2020-02-04 16:16:21 +00:00
proc newMultistream*(): MultistreamSelect =
2019-08-30 05:16:55 +00:00
new result
2019-09-06 21:25:54 +00:00
result.codec = MSCodec
2019-08-30 05:16:55 +00:00
template validateSuffix(str: string): untyped =
if str.endsWith("\n"):
str.removeSuffix("\n")
else:
raise newException(CatchableError, "MultistreamSelect failed, malformed message")
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect,
2019-08-30 05:16:55 +00:00
conn: Connection,
2019-09-14 13:55:52 +00:00
proto: seq[string]):
Future[string] {.async.} =
trace "initiating handshake", codec = m.codec
2019-08-30 05:16:55 +00:00
## select a remote protocol
2019-09-05 15:19:15 +00:00
await conn.write(m.codec) # write handshake
2019-08-30 05:16:55 +00:00
if proto.len() > 0:
trace "selecting proto", proto = proto[0]
2019-09-05 15:19:15 +00:00
await conn.writeLp((proto[0] & "\n")) # select proto
2019-08-30 05:16:55 +00:00
var s = string.fromBytes((await conn.readLp(1024))) # read ms header
validateSuffix(s)
if s != Codec:
notice "handshake failed", codec = s
raise newException(CatchableError, "MultistreamSelect handshake failed")
else:
trace "multistream handshake success"
2019-08-30 05:16:55 +00:00
if proto.len() == 0: # no protocols, must be a handshake call
return Codec
else:
s = string.fromBytes(await conn.readLp(1024)) # read the first proto
validateSuffix(s)
trace "reading first requested proto"
if s == proto[0]:
trace "successfully selected ", proto = proto[0]
return proto[0]
elif proto.len > 1:
# Try to negotiate alternatives
let protos = proto[1..<proto.len()]
trace "selecting one of several protos", protos = protos
for p in protos:
trace "selecting proto", proto = p
await conn.writeLp((p & "\n")) # select proto
s = string.fromBytes(await conn.readLp(1024)) # read the first proto
validateSuffix(s)
if s == p:
trace "selected protocol", protocol = s
return s
return ""
else:
# No alternatives, fail
return ""
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect,
conn: Connection,
2020-01-07 08:02:37 +00:00
proto: string): Future[bool] {.async.} =
if proto.len > 0:
return (await m.select(conn, @[proto])) == proto
2020-01-07 08:02:37 +00:00
else:
return (await m.select(conn, @[])) == Codec
2020-02-04 16:16:21 +00:00
proc select*(m: MultistreamSelect, conn: Connection): Future[bool] =
2019-09-08 07:43:33 +00:00
m.select(conn, "")
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
proc list*(m: MultistreamSelect,
2019-08-30 05:16:55 +00:00
conn: Connection): Future[seq[string]] {.async.} =
## list remote protos requests on connection
if not await m.select(conn):
2019-08-30 05:16:55 +00:00
return
await conn.write(Ls) # send ls
2019-08-30 05:16:55 +00:00
var list = newSeq[string]()
2020-05-08 20:58:23 +00:00
let ms = string.fromBytes(await conn.readLp(1024))
2019-08-30 05:16:55 +00:00
for s in ms.split("\n"):
if s.len() > 0:
list.add(s)
result = list
proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.async, gcsafe.} =
trace "handle: starting multistream handling", handshaked = active
var handshaked = active
2020-05-23 17:10:22 +00:00
try:
while not conn.closed:
2020-05-08 20:58:23 +00:00
var ms = string.fromBytes(await conn.readLp(1024))
validateSuffix(ms)
if not handshaked and ms != Codec:
error "expected handshake message", instead=ms
2020-08-02 10:22:49 +00:00
raise newException(CatchableError,
"MultistreamSelect handling failed, invalid first message")
2020-01-07 08:02:37 +00:00
2020-08-02 10:22:49 +00:00
trace "handle: got request", ms
if ms.len() <= 0:
2019-09-14 15:55:58 +00:00
trace "handle: invalid proto"
await conn.write(Na)
if m.handlers.len() == 0:
2020-08-02 10:22:49 +00:00
trace "handle: sending `na` for protocol", protocol = ms
await conn.write(Na)
continue
case ms:
of "ls":
trace "handle: listing protos"
var protos = ""
for h in m.handlers:
protos &= (h.proto & "\n")
await conn.writeLp(protos)
of Codec:
if not handshaked:
await conn.write(m.codec)
handshaked = true
else:
trace "handle: sending `na` for duplicate handshake while handshaked"
await conn.write(Na)
else:
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or ms == h.proto:
2020-08-02 10:22:49 +00:00
trace "found handler", protocol = ms
await conn.writeLp((h.proto & "\n"))
await h.protocol.handler(conn, ms)
return
2020-08-02 10:22:49 +00:00
debug "no handlers", protocol = ms
await conn.write(Na)
except CancelledError as exc:
await conn.close()
raise exc
2020-05-23 17:10:22 +00:00
except CatchableError as exc:
trace "exception in multistream", exc = exc.msg
await conn.close()
2020-05-23 17:10:22 +00:00
finally:
trace "leaving multistream loop"
2019-08-30 05:16:55 +00:00
2020-02-04 16:16:21 +00:00
proc addHandler*[T: LPProtocol](m: MultistreamSelect,
2019-09-03 20:40:19 +00:00
codec: string,
2019-08-30 05:16:55 +00:00
protocol: T,
matcher: Matcher = nil) =
2019-09-14 13:55:52 +00:00
## register a protocol
2020-01-07 08:02:37 +00:00
# TODO: This is a bug in chronicles,
# it break if I uncomment this line.
2020-01-07 08:02:37 +00:00
# Which is almost the same as the
2019-09-14 13:55:52 +00:00
# one on the next override of addHandler
#
2019-09-14 15:55:58 +00:00
# trace "registering protocol", codec = codec
2019-09-14 13:55:52 +00:00
m.handlers.add(HandlerHolder(proto: codec,
protocol: protocol,
match: matcher))
2020-02-04 16:16:21 +00:00
proc addHandler*[T: LPProtoHandler](m: MultistreamSelect,
2019-09-14 13:55:52 +00:00
codec: string,
handler: T,
matcher: Matcher = nil) =
## helper to allow registering pure handlers
2019-09-14 15:55:58 +00:00
trace "registering proto handler", codec = codec
2019-09-14 13:55:52 +00:00
let protocol = new LPProtocol
protocol.codec = codec
protocol.handler = handler
2019-09-03 20:40:19 +00:00
m.handlers.add(HandlerHolder(proto: codec,
2019-08-30 05:16:55 +00:00
protocol: protocol,
match: matcher))