vacp2p
/
nim-libp2p
mirror of https://github.com/vacp2p/nim-libp2p.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix varint issues 

* fixes #111
This commit is contained in:
Jacek Sieka 2020-05-08 22:58:23 +02:00 committed by Dmitriy Ryajov
parent 87e1f3c61f
commit 3053f03814
20 changed files with 143 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/GETTING_STARTED.md
View File

@ -34,7 +34,7 @@ type
method init(p: TestProto) {.gcsafe.} = method init(p: TestProto) {.gcsafe.} =
# handle incoming connections in closure # handle incoming connections in closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} = proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Got from remote - ", cast[string](await conn.readLp()) echo "Got from remote - ", cast[string](await conn.readLp(1024))
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
await conn.close() await conn.close()
@ -90,7 +90,7 @@ proc main() {.async, gcsafe.} =
await conn.writeLp("Hello!") # writeLp send a length prefixed buffer over the wire await conn.writeLp("Hello!") # writeLp send a length prefixed buffer over the wire
# readLp reads length prefixed bytes and returns a buffer without the prefix # readLp reads length prefixed bytes and returns a buffer without the prefix
echo "Remote responded with - ", cast[string](await conn.readLp()) echo "Remote responded with - ", cast[string](await conn.readLp(1024))
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
await allFutures(switch1Fut & switch2Fut) # wait for all transports to shutdown await allFutures(switch1Fut & switch2Fut) # wait for all transports to shutdown

45
docs/tutorial/directchat/second.nim
View File

@ -2,23 +2,23 @@ when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".} {.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils import tables, strformat, strutils
import chronos import chronos
import ../libp2p/[switch, import ../libp2p/[switch,
multistream, multistream,
crypto/crypto, crypto/crypto,
protocols/identify, protocols/identify,
connection, connection,
transports/transport, transports/transport,
transports/tcptransport, transports/tcptransport,
multiaddress, multiaddress,
peerinfo, peerinfo,
peer, peer,
protocols/protocol, protocols/protocol,
protocols/secure/secure, protocols/secure/secure,
protocols/secure/secio, protocols/secure/secio,
muxers/muxer, muxers/muxer,
muxers/mplex/mplex, muxers/mplex/mplex,
muxers/mplex/types] muxers/mplex/types]
const ChatCodec = "/nim-libp2p/chat/1.0.0" const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505" const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
@ -33,12 +33,12 @@ const Help = """
type ChatProto = ref object of LPProtocol type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer connected: bool # if the node is connected to another peer
started: bool # if the node has started started: bool # if the node has started
# copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115 # copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T = proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str) let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address): if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
@ -60,7 +60,7 @@ proc dialPeer(p: ChatProto, address: string) {.async.} =
proc readAndPrint(p: ChatProto) {.async.} = proc readAndPrint(p: ChatProto) {.async.} =
while true: while true:
while p.connected: while p.connected:
echo cast[string](await p.conn.readLp()) echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis) await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} = proc writeAndPrint(p: ChatProto) {.async.} =
@ -140,12 +140,11 @@ proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe() let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe: if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!") raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD] var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd) thread.createThread(readInput, wfd)
await processInput(rfd) await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main()) waitFor(main())

45
docs/tutorial/second.nim
View File

@ -2,23 +2,23 @@ when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".} {.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils import tables, strformat, strutils
import chronos import chronos
import ../libp2p/[switch, import ../libp2p/[switch,
multistream, multistream,
crypto/crypto, crypto/crypto,
protocols/identify, protocols/identify,
connection, connection,
transports/transport, transports/transport,
transports/tcptransport, transports/tcptransport,
multiaddress, multiaddress,
peerinfo, peerinfo,
peer, peer,
protocols/protocol, protocols/protocol,
protocols/secure/secure, protocols/secure/secure,
protocols/secure/secio, protocols/secure/secio,
muxers/muxer, muxers/muxer,
muxers/mplex/mplex, muxers/mplex/mplex,
muxers/mplex/types] muxers/mplex/types]
const ChatCodec = "/nim-libp2p/chat/1.0.0" const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505" const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
@ -33,12 +33,12 @@ const Help = """
type ChatProto = ref object of LPProtocol type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer connected: bool # if the node is connected to another peer
started: bool # if the node has started started: bool # if the node has started
# copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115 # copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T = proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str) let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address): if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
@ -60,7 +60,7 @@ proc dialPeer(p: ChatProto, address: string) {.async.} =
proc readAndPrint(p: ChatProto) {.async.} = proc readAndPrint(p: ChatProto) {.async.} =
while true: while true:
while p.connected: while p.connected:
echo cast[string](await p.conn.readLp()) echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis) await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} = proc writeAndPrint(p: ChatProto) {.async.} =
@ -140,12 +140,11 @@ proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe() let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe: if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!") raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD] var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd) thread.createThread(readInput, wfd)
await processInput(rfd) await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main()) waitFor(main())

10
examples/directchat.nim
View File

@ -12,14 +12,14 @@ import ../libp2p/[switch, # manage transports, a single entry
transports/transport, # listen and dial to other peers using p2p protocol transports/transport, # listen and dial to other peers using p2p protocol
transports/tcptransport, # listen and dial to other peers using client-server protocol transports/tcptransport, # listen and dial to other peers using client-server protocol
multiaddress, # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP multiaddress, # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
peerinfo, # manage the information of a peer, such as peer ID and public / private key peerinfo, # manage the information of a peer, such as peer ID and public / private key
peer, # Implement how peers interact peer, # Implement how peers interact
protocols/protocol, # define the protocol base type protocols/protocol, # define the protocol base type
protocols/secure/secure, # define the protocol of secure connection protocols/secure/secure, # define the protocol of secure connection
protocols/secure/secio, # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS protocols/secure/secio, # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
muxers/mplex/mplex, # implement stream multiplexing muxers/mplex/mplex, # implement stream multiplexing
muxers/mplex/types] # define some contants and message types for stream multiplexing muxers/mplex/types] # define some contants and message types for stream multiplexing
const ChatCodec = "/nim-libp2p/chat/1.0.0" const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505" const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
@ -34,7 +34,7 @@ const Help = """
type ChatProto = ref object of LPProtocol type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer connected: bool # if the node is connected to another peer
started: bool # if the node has started started: bool # if the node has started
@ -63,7 +63,7 @@ proc readAndPrint(p: ChatProto) {.async.} =
while p.connected: while p.connected:
# TODO: echo &"{p.id} -> " # TODO: echo &"{p.id} -> "
echo cast[string](await p.conn.readLp()) echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis) await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} = proc writeAndPrint(p: ChatProto) {.async.} =

46
libp2p/connection.nim
View File

@ -23,7 +23,6 @@ logScope:
topic = "Connection" topic = "Connection"
const const
DefaultReadSize* = 1 shl 20
ConnectionTrackerName* = "libp2p.connection" ConnectionTrackerName* = "libp2p.connection"
type type
@ -35,9 +34,6 @@ type
# (we got many actually :-)) # (we got many actually :-))
readLoops*: seq[Future[void]] readLoops*: seq[Future[void]]
InvalidVarintException = object of LPStreamError
InvalidVarintSizeException = object of LPStreamError
ConnectionTracker* = ref object of TrackerBase ConnectionTracker* = ref object of TrackerBase
opened*: uint64 opened*: uint64
closed*: uint64 closed*: uint64
@ -68,12 +64,6 @@ proc setupConnectionTracker(): ConnectionTracker =
declareGauge libp2p_open_connection, "open Connection instances" declareGauge libp2p_open_connection, "open Connection instances"
proc newInvalidVarintException*(): ref InvalidVarintException =
newException(InvalidVarintException, "Unable to parse varint")
proc newInvalidVarintSizeException*(): ref InvalidVarintSizeException =
newException(InvalidVarintSizeException, "Wrong varint size")
proc bindStreamClose(conn: Connection) {.async.} = proc bindStreamClose(conn: Connection) {.async.} =
# bind stream's close event to connection's close # bind stream's close event to connection's close
# to ensure correct close propagation # to ensure correct close propagation
@ -155,42 +145,6 @@ method close*(s: Connection) {.async, gcsafe.} =
s.peerInfo.id else: "" s.peerInfo.id else: ""
libp2p_open_connection.dec() libp2p_open_connection.dec()
proc readLp*(s: Connection): Future[seq[byte]] {.async, gcsafe.} =
## read lenght prefixed msg
var
size: uint
length: int
res: VarintStatus
buff = newSeq[byte](10)
try:
for i in 0..<len(buff):
await s.readExactly(addr buff[i], 1)
res = LP.getUVarint(buff.toOpenArray(0, i), length, size)
if res == VarintStatus.Success:
break
if res != VarintStatus.Success:
raise newInvalidVarintException()
if size.int > DefaultReadSize:
raise newInvalidVarintSizeException()
buff.setLen(size)
if size > 0.uint:
trace "reading exact bytes from stream", size = size
await s.readExactly(addr buff[0], int(size))
return buff
except LPStreamIncompleteError as exc:
trace "remote connection ended unexpectedly", exc = exc.msg
raise exc
except LPStreamReadError as exc:
trace "couldn't read from stream", exc = exc.msg
raise exc
proc writeLp*(s: Connection, msg: string | seq[byte]): Future[void] {.gcsafe.} =
## write lenght prefixed
var buf = initVBuffer()
buf.writeSeq(msg)
buf.finish()
s.write(buf.buffer)
method getObservedAddrs*(c: Connection): Future[MultiAddress] {.base, async, gcsafe.} = method getObservedAddrs*(c: Connection): Future[MultiAddress] {.base, async, gcsafe.} =
## get resolved multiaddresses for the connection ## get resolved multiaddresses for the connection
result = c.observedAddrs result = c.observedAddrs

10
libp2p/multistream.nim
View File

@ -58,7 +58,7 @@ proc select*(m: MultistreamSelect,
trace "selecting proto", proto = proto trace "selecting proto", proto = proto
await conn.writeLp((proto[0] & "\n")) # select proto await conn.writeLp((proto[0] & "\n")) # select proto
result = cast[string]((await conn.readLp())) # read ms header result = cast[string]((await conn.readLp(1024))) # read ms header
result.removeSuffix("\n") result.removeSuffix("\n")
if result != Codec: if result != Codec:
error "handshake failed", codec = result.toHex() error "handshake failed", codec = result.toHex()
@ -67,7 +67,7 @@ proc select*(m: MultistreamSelect,
if proto.len() == 0: # no protocols, must be a handshake call if proto.len() == 0: # no protocols, must be a handshake call
return return
result = string.fromBytes(await conn.readLp()) # read the first proto result = string.fromBytes(await conn.readLp(1024)) # read the first proto
trace "reading first requested proto" trace "reading first requested proto"
result.removeSuffix("\n") result.removeSuffix("\n")
if result == proto[0]: if result == proto[0]:
@ -78,7 +78,7 @@ proc select*(m: MultistreamSelect,
trace "selecting one of several protos" trace "selecting one of several protos"
for p in proto[1..<proto.len()]: for p in proto[1..<proto.len()]:
await conn.writeLp((p & "\n")) # select proto await conn.writeLp((p & "\n")) # select proto
result = string.fromBytes(await conn.readLp()) # read the first proto result = string.fromBytes(await conn.readLp(1024)) # read the first proto
result.removeSuffix("\n") result.removeSuffix("\n")
if result == p: if result == p:
trace "selected protocol", protocol = result trace "selected protocol", protocol = result
@ -104,7 +104,7 @@ proc list*(m: MultistreamSelect,
await conn.write(Ls) # send ls await conn.write(Ls) # send ls
var list = newSeq[string]() var list = newSeq[string]()
let ms = string.fromBytes(await conn.readLp()) let ms = string.fromBytes(await conn.readLp(1024))
for s in ms.split("\n"): for s in ms.split("\n"):
if s.len() > 0: if s.len() > 0:
list.add(s) list.add(s)
@ -115,7 +115,7 @@ proc handle*(m: MultistreamSelect, conn: Connection) {.async, gcsafe.} =
trace "handle: starting multistream handling" trace "handle: starting multistream handling"
tryAndWarn "multistream handle": tryAndWarn "multistream handle":
while not conn.closed: while not conn.closed:
var ms = string.fromBytes(await conn.readLp()) var ms = string.fromBytes(await conn.readLp(1024))
ms.removeSuffix("\n") ms.removeSuffix("\n")
trace "handle: got request for ", ms trace "handle: got request for ", ms

37
libp2p/muxers/mplex/coder.nim
View File

@ -11,6 +11,7 @@ import chronos
import nimcrypto/utils, chronicles import nimcrypto/utils, chronicles
import types, import types,
../../connection, ../../connection,
../../utility,
../../varint, ../../varint,
../../vbuffer, ../../vbuffer,
../../stream/lpstream ../../stream/lpstream
@ -29,46 +30,18 @@ type
proc newInvalidMplexMsgType*(): ref InvalidMplexMsgType = proc newInvalidMplexMsgType*(): ref InvalidMplexMsgType =
newException(InvalidMplexMsgType, "invalid message type") newException(InvalidMplexMsgType, "invalid message type")
proc readMplexVarint(conn: Connection): Future[uint64] {.async, gcsafe.} =
var
varint: uint
length: int
res: VarintStatus
buffer = newSeq[byte](10)
try:
for i in 0..<len(buffer):
await conn.readExactly(addr buffer[i], 1)
res = PB.getUVarint(buffer.toOpenArray(0, i), length, varint)
if res == VarintStatus.Success:
break
if res != VarintStatus.Success:
raise newInvalidVarintException()
return varint
except LPStreamIncompleteError as exc:
trace "unable to read varint", exc = exc.msg
raise exc
proc readMsg*(conn: Connection): Future[Msg] {.async, gcsafe.} = proc readMsg*(conn: Connection): Future[Msg] {.async, gcsafe.} =
let header = await conn.readMplexVarint() let header = await conn.readVarint()
trace "read header varint", varint = header trace "read header varint", varint = header
let dataLenVarint = await conn.readMplexVarint() let data = await conn.readLp(MaxMsgSize)
trace "read data len varint", varint = dataLenVarint trace "read data", dataLen = data.len, data = shortLog(data)
if dataLenVarint.int > DefaultReadSize:
raise newInvalidVarintSizeException()
var data: seq[byte] = newSeq[byte](dataLenVarint.int)
if dataLenVarint.int > 0:
await conn.readExactly(addr data[0], dataLenVarint.int)
trace "read data", data = data.len
let msgType = header and 0x7 let msgType = header and 0x7
if msgType.int > ord(MessageType.ResetOut): if msgType.int > ord(MessageType.ResetOut):
raise newInvalidMplexMsgType() raise newInvalidMplexMsgType()
result = (uint64(header shr 3), MessageType(msgType), data) result = (header shr 3, MessageType(msgType), data)
proc writeMsg*(conn: Connection, proc writeMsg*(conn: Connection,
id: uint64, id: uint64,

1
libp2p/muxers/mplex/types.nim
View File

@ -9,6 +9,7 @@
import chronos import chronos
# https://github.com/libp2p/specs/tree/master/mplex#writing-to-a-stream
const MaxMsgSize* = 1 shl 20 # 1mb const MaxMsgSize* = 1 shl 20 # 1mb
const MaxChannels* = 1000 const MaxChannels* = 1000
const MplexCodec* = "/mplex/6.7.0" const MplexCodec* = "/mplex/6.7.0"

2
libp2p/protocols/identify.nim
View File

@ -124,7 +124,7 @@ proc identify*(p: Identify,
conn: Connection, conn: Connection,
remotePeerInfo: PeerInfo): Future[IdentifyInfo] {.async, gcsafe.} = remotePeerInfo: PeerInfo): Future[IdentifyInfo] {.async, gcsafe.} =
trace "initiating identify" trace "initiating identify"
var message = await conn.readLp() var message = await conn.readLp(64*1024)
if len(message) == 0: if len(message) == 0:
trace "identify: Invalid or empty message received!" trace "identify: Invalid or empty message received!"
raise newException(IdentityInvalidMsgError, raise newException(IdentityInvalidMsgError,

2
libp2p/protocols/pubsub/pubsubpeer.nim
View File

@ -56,7 +56,7 @@ proc handle*(p: PubSubPeer, conn: Connection) {.async.} =
try: try:
while not conn.closed: while not conn.closed:
trace "waiting for data", peer = p.id, closed = conn.closed trace "waiting for data", peer = p.id, closed = conn.closed
let data = await conn.readLp() let data = await conn.readLp(64 * 1024)
let hexData = data.toHex() let hexData = data.toHex()
trace "read data from peer", peer = p.id, data = data.shortLog trace "read data from peer", peer = p.id, data = data.shortLog
if $hexData.hash in p.recvdRpcCache: if $hexData.hash in p.recvdRpcCache:

46
libp2p/stream/lpstream.nim
View File

@ -8,7 +8,9 @@
## those terms. ## those terms.
import oids import oids
import chronicles, chronos import chronicles, chronos, strformat
import ../varint,
../vbuffer
type type
LPStream* = ref object of RootObj LPStream* = ref object of RootObj
@ -27,6 +29,9 @@ type
par*: ref Exception par*: ref Exception
LPStreamEOFError* = object of LPStreamError LPStreamEOFError* = object of LPStreamError
InvalidVarintError* = object of LPStreamError
MaxSizeError* = object of LPStreamError
proc newLPStreamReadError*(p: ref Exception): ref Exception = proc newLPStreamReadError*(p: ref Exception): ref Exception =
var w = newException(LPStreamReadError, "Read stream failed") var w = newException(LPStreamReadError, "Read stream failed")
w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg
@ -109,6 +114,45 @@ proc readLine*(s: LPStream, limit = 0, sep = "\r\n"): Future[string] {.async, de
except LPStreamIncompleteError, LPStreamReadError: except LPStreamIncompleteError, LPStreamReadError:
discard # EOF, in which case we should return whatever we read so far.. discard # EOF, in which case we should return whatever we read so far..
proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe.} =
var
varint: uint64
length: int
buffer: array[10, byte]
for i in 0..<len(buffer):
await conn.readExactly(addr buffer[i], 1)
let res = PB.getUVarint(buffer.toOpenArray(0, i), length, varint)
if res == VarintStatus.Success:
return varint
if res != VarintStatus.Incomplete:
break
if true: # can't end with a raise apparently
raise (ref InvalidVarintError)(msg: "Cannot parse varint")
proc readLp*(s: LPStream, maxSize: int): Future[seq[byte]] {.async, gcsafe.} =
## read length prefixed msg, with the length encoded as a varint
let
length = await s.readVarint()
maxLen = uint64(if maxSize < 0: int.high else: maxSize)
if length > maxLen:
raise (ref MaxSizeError)(msg: "Message exceeds maximum length")
if length == 0:
return
var res = newSeq[byte](length)
await s.readExactly(addr res[0], res.len)
return res
proc writeLp*(s: LPStream, msg: string | seq[byte]): Future[void] {.gcsafe.} =
## write length prefixed
var buf = initVBuffer()
buf.writeSeq(msg)
buf.finish()
s.write(buf.buffer)
method write*(s: LPStream, msg: seq[byte]) {.base, async.} = method write*(s: LPStream, msg: seq[byte]) {.base, async.} =
doAssert(false, "not implemented!") doAssert(false, "not implemented!")

2
tests/pubsub/testfloodsub.nim
View File

@ -7,6 +7,8 @@
## This file may not be copied, modified, or distributed except according to ## This file may not be copied, modified, or distributed except according to
## those terms. ## those terms.
{.used.}
import unittest, sequtils, options, tables, sets import unittest, sequtils, options, tables, sets
import chronos import chronos
import utils, import utils,

2
tests/pubsub/testgossipinternal.nim
View File

@ -1,5 +1,7 @@
include ../../libp2p/protocols/pubsub/gossipsub include ../../libp2p/protocols/pubsub/gossipsub
{.used.}
import unittest import unittest
import ../../libp2p/errors import ../../libp2p/errors
import ../../libp2p/stream/bufferstream import ../../libp2p/stream/bufferstream

2
tests/pubsub/testgossipsub.nim
View File

@ -7,6 +7,8 @@
## This file may not be copied, modified, or distributed except according to ## This file may not be copied, modified, or distributed except according to
## those terms. ## those terms.
{.used.}
import unittest, sequtils, options, tables, sets import unittest, sequtils, options, tables, sets
import chronos import chronos
import chronicles import chronicles

2
tests/pubsub/testmcache.nim
View File

@ -1,3 +1,5 @@
{.used.}
import options, sets, sequtils import options, sets, sequtils
import unittest import unittest
import ../../libp2p/[peer, import ../../libp2p/[peer,

14
tests/testinterop.nim
View File

@ -55,7 +55,7 @@ proc readLp*(s: StreamTransport): Future[seq[byte]] {.async, gcsafe.} =
if res == VarintStatus.Success: if res == VarintStatus.Success:
break break
if res != VarintStatus.Success: if res != VarintStatus.Success:
raise newInvalidVarintException() raise (ref InvalidVarintError)()
result.setLen(size) result.setLen(size)
if size > 0.uint: if size > 0.uint:
await s.readExactly(addr result[0], int(size)) await s.readExactly(addr result[0], int(size))
@ -209,11 +209,11 @@ suite "Interop":
daemonPeer.addresses), daemonPeer.addresses),
protos[0]) protos[0])
await conn.writeLp("test 1") await conn.writeLp("test 1")
check "test 2" == cast[string]((await conn.readLp())) check "test 2" == cast[string]((await conn.readLp(1024)))
await sleepAsync(10.millis) await sleepAsync(10.millis)
await conn.writeLp("test 3") await conn.writeLp("test 3")
check "test 4" == cast[string]((await conn.readLp())) check "test 4" == cast[string]((await conn.readLp(1024)))
await wait(testFuture, 10.secs) await wait(testFuture, 10.secs)
await nativeNode.stop() await nativeNode.stop()
@ -271,7 +271,7 @@ suite "Interop":
var testFuture = newFuture[string]("test.future") var testFuture = newFuture[string]("test.future")
proc nativeHandler(conn: Connection, proto: string) {.async.} = proc nativeHandler(conn: Connection, proto: string) {.async.} =
var line = cast[string](await conn.readLp()) var line = cast[string](await conn.readLp(1024))
check line == test check line == test
testFuture.complete(line) testFuture.complete(line)
await conn.close() await conn.close()
@ -306,10 +306,10 @@ suite "Interop":
var testFuture = newFuture[void]("test.future") var testFuture = newFuture[void]("test.future")
proc nativeHandler(conn: Connection, proto: string) {.async.} = proc nativeHandler(conn: Connection, proto: string) {.async.} =
check "test 1" == cast[string](await conn.readLp()) check "test 1" == cast[string](await conn.readLp(1024))
await conn.writeLp(cast[seq[byte]]("test 2")) await conn.writeLp(cast[seq[byte]]("test 2"))
check "test 3" == cast[string](await conn.readLp()) check "test 3" == cast[string](await conn.readLp(1024))
await conn.writeLp(cast[seq[byte]]("test 4")) await conn.writeLp(cast[seq[byte]]("test 4"))
testFuture.complete() testFuture.complete()
@ -355,7 +355,7 @@ suite "Interop":
var testFuture = newFuture[int]("test.future") var testFuture = newFuture[int]("test.future")
proc nativeHandler(conn: Connection, proto: string) {.async.} = proc nativeHandler(conn: Connection, proto: string) {.async.} =
while count < 10: while count < 10:
var line = cast[string](await conn.readLp()) var line = cast[string](await conn.readLp(1024))
check line == test check line == test
await conn.writeLp(cast[seq[byte]](test)) await conn.writeLp(cast[seq[byte]](test))
count.inc() count.inc()

18
tests/testmplex.nim
View File

@ -153,7 +153,7 @@ suite "Mplex":
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(1024)
check cast[string](msg) == "Hello from stream!" check cast[string](msg) == "Hello from stream!"
await stream.close() await stream.close()
done.complete() done.complete()
@ -200,7 +200,7 @@ suite "Mplex":
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(1024)
check cast[string](msg) == "Hello from stream!" check cast[string](msg) == "Hello from stream!"
await stream.close() await stream.close()
done.complete() done.complete()
@ -251,7 +251,7 @@ suite "Mplex":
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
defer: defer:
await stream.close() await stream.close()
let msg = await stream.readLp() let msg = await stream.readLp(MaxMsgSize)
check msg == bigseq check msg == bigseq
trace "Bigseq check passed!" trace "Bigseq check passed!"
listenJob.complete() listenJob.complete()
@ -312,7 +312,7 @@ suite "Mplex":
let mplexDial = newMplex(conn) let mplexDial = newMplex(conn)
let dialFut = mplexDial.handle() let dialFut = mplexDial.handle()
let stream = await mplexDial.newStream("DIALER") let stream = await mplexDial.newStream("DIALER")
let msg = cast[string](await stream.readLp()) let msg = cast[string](await stream.readLp(1024))
check msg == "Hello from stream!" check msg == "Hello from stream!"
# await dialFut # await dialFut
@ -339,7 +339,7 @@ suite "Mplex":
var listenConn: Connection var listenConn: Connection
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(1024)
check cast[string](msg) == &"stream {count}!" check cast[string](msg) == &"stream {count}!"
count.inc count.inc
await stream.close() await stream.close()
@ -386,7 +386,7 @@ suite "Mplex":
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
listenConn = conn listenConn = conn
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(1024)
check cast[string](msg) == &"stream {count} from dialer!" check cast[string](msg) == &"stream {count} from dialer!"
await stream.writeLp(&"stream {count} from listener!") await stream.writeLp(&"stream {count} from listener!")
count.inc count.inc
@ -411,7 +411,7 @@ suite "Mplex":
for i in 1..10: for i in 1..10:
let stream = await mplexDial.newStream("dialer stream") let stream = await mplexDial.newStream("dialer stream")
await stream.writeLp(&"stream {i} from dialer!") await stream.writeLp(&"stream {i} from dialer!")
let msg = await stream.readLp() let msg = await stream.readLp(1024)
check cast[string](msg) == &"stream {i} from listener!" check cast[string](msg) == &"stream {i} from listener!"
await stream.close() await stream.close()
@ -473,7 +473,7 @@ suite "Mplex":
const MsgSize = 1024 const MsgSize = 1024
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(MsgSize)
check msg.len == MsgSize check msg.len == MsgSize
await stream.close() await stream.close()
complete.complete() complete.complete()
@ -542,7 +542,7 @@ suite "Mplex":
const MsgSize = 512 const MsgSize = 512
proc connHandler(conn: Connection) {.async, gcsafe.} = proc connHandler(conn: Connection) {.async, gcsafe.} =
proc handleMplexListen(stream: Connection) {.async, gcsafe.} = proc handleMplexListen(stream: Connection) {.async, gcsafe.} =
let msg = await stream.readLp() let msg = await stream.readLp(MsgSize)
check msg.len == MsgSize check msg.len == MsgSize
await stream.close() await stream.close()
complete.complete() complete.complete()

6
tests/testmultistream.nim
View File

@ -269,7 +269,7 @@ suite "Multistream select":
check (await msDial.select(conn, "/test/proto/1.0.0")) == true check (await msDial.select(conn, "/test/proto/1.0.0")) == true
let hello = cast[string](await conn.readLp()) let hello = cast[string](await conn.readLp(1024))
result = hello == "Hello!" result = hello == "Hello!"
await conn.close() await conn.close()
@ -358,7 +358,7 @@ suite "Multistream select":
check (await msDial.select(conn, check (await msDial.select(conn,
@["/test/proto/1.0.0", "/test/no/proto/1.0.0"])) == "/test/proto/1.0.0" @["/test/proto/1.0.0", "/test/no/proto/1.0.0"])) == "/test/proto/1.0.0"
let hello = cast[string](await conn.readLp()) let hello = cast[string](await conn.readLp(1024))
result = hello == "Hello!" result = hello == "Hello!"
await conn.close() await conn.close()
@ -397,7 +397,7 @@ suite "Multistream select":
check (await msDial.select(conn, @["/test/proto2/1.0.0", "/test/proto1/1.0.0"])) == "/test/proto2/1.0.0" check (await msDial.select(conn, @["/test/proto2/1.0.0", "/test/proto1/1.0.0"])) == "/test/proto2/1.0.0"
result = cast[string](await conn.readLp()) == "Hello from /test/proto2/1.0.0!" result = cast[string](await conn.readLp(1024)) == "Hello from /test/proto2/1.0.0!"
await conn.close() await conn.close()
await transport2.close() await transport2.close()

6
tests/testnoise.nim
View File

@ -41,7 +41,7 @@ type
method init(p: TestProto) {.gcsafe.} = method init(p: TestProto) {.gcsafe.} =
proc handle(conn: Connection, proto: string) {.async, gcsafe.} = proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
await conn.close() await conn.close()
@ -166,7 +166,7 @@ suite "Noise":
let sconn = await serverNoise.secure(conn, false) let sconn = await serverNoise.secure(conn, false)
defer: defer:
await sconn.close() await sconn.close()
let msg = await sconn.readLp() let msg = await sconn.readLp(1024*1024)
check msg == hugePayload check msg == hugePayload
readTask.complete() readTask.complete()
@ -214,7 +214,7 @@ suite "Noise":
awaiters.add(await switch2.start()) awaiters.add(await switch2.start())
let conn = await switch2.dial(switch1.peerInfo, TestCodec) let conn = await switch2.dial(switch1.peerInfo, TestCodec)
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
await allFuturesThrowing(switch1.stop(), switch2.stop()) await allFuturesThrowing(switch1.stop(), switch2.stop())

8
tests/testswitch.nim
View File

@ -68,7 +68,7 @@ suite "Switch":
let done = newFuture[void]() let done = newFuture[void]()
proc handle(conn: Connection, proto: string) {.async, gcsafe.} = proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
await conn.close() await conn.close()
@ -87,7 +87,7 @@ suite "Switch":
try: try:
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
result = true result = true
except LPStreamError: except LPStreamError:
@ -118,7 +118,7 @@ suite "Switch":
(switch1, peerInfo1) = createSwitch(ma1) (switch1, peerInfo1) = createSwitch(ma1)
proc handle(conn: Connection, proto: string) {.async, gcsafe.} = proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
await conn.close() await conn.close()
@ -136,7 +136,7 @@ suite "Switch":
try: try:
await conn.writeLp("Hello!") await conn.writeLp("Hello!")
let msg = cast[string](await conn.readLp()) let msg = cast[string](await conn.readLp(1024))
check "Hello!" == msg check "Hello!" == msg
result = true result = true
except LPStreamError: except LPStreamError: