Dmitriy Ryajov e623e70e7b
PubSub (Gossip & Flood) Implementation (#36)
This adds gossipsub and floodsub, as well as basic interop testing with the go libp2p daemon. 

* add close event

* wip: gossipsub

* splitting rpc message

* making message handling more consistent

* initial gossipsub implementation

* feat: nim 1.0 cleanup

* wip: gossipsub protobuf

* adding encoding/decoding of gossipsub messages

* add disconnect handler

* add proper gossipsub msg handling

* misc: cleanup for nim 1.0

* splitting floodsub and gossipsub tests

* feat: add mesh rebalansing

* test pubsub

* add mesh rebalansing tests

* testing mesh maintenance

* finishing mcache implementatin

* wip: commenting out broken tests

* wip: don't run heartbeat for now

* switchout debug for trace logging

* testing gossip peer selection algorithm

* test stream piping

* more work around message amplification

* get the peerid from message

* use timed cache as backing store

* allow setting timeout in constructor

* several changes to improve performance

* more through testing of msg amplification

* prevent gc issues

* allow piping to self and prevent deadlocks

* improove floodsub

* allow running hook on cache eviction

* prevent race conditions

* prevent race conditions and improove tests

* use hashes as cache keys

* removing useless file

* don't create a new seq

* re-enable pubsub tests

* fix imports

* reduce number of runs to speed up tests

* break out control message processing

* normalize sleeps between steps

* implement proper transport filtering

* initial interop testing

* clean up floodsub publish logic

* allow dialing without a protocol

* adding multiple reads/writes

* use protobuf varint in mplex

* don't loose conn's peerInfo

* initial interop pubsub tests

* don't duplicate connections/peers

* bring back interop tests

* wip: interop

* re-enable interop and daemon tests

* add multiple read write tests from handlers

* don't cleanup channel prematurely

* use correct channel to send/receive msgs

* adjust tests with latest changes

* include interop tests

* remove temp logging output

* fix ci

* use correct public key serialization

* additional tests for pubsub interop
2019-12-05 20:16:18 -06:00

155 lines
5.3 KiB
Nim

## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
## TODO:
## Timeouts and message limits are still missing
## they need to be added ASAP
import tables, sequtils, options
import chronos, chronicles
import ../muxer,
../../connection,
../../stream/lpstream,
coder,
types,
lpchannel
logScope:
topic = "Mplex"
type
Mplex* = ref object of Muxer
remote*: Table[uint, LPChannel]
local*: Table[uint, LPChannel]
currentId*: uint
maxChannels*: uint
proc getChannelList(m: Mplex, initiator: bool): var Table[uint, LPChannel] =
if initiator:
trace "picking local channels", initiator = initiator
result = m.local
else:
trace "picking remote channels", initiator = initiator
result = m.remote
proc newStreamInternal*(m: Mplex,
initiator: bool = true,
chanId: uint = 0,
name: string = ""):
Future[LPChannel] {.async, gcsafe.} =
## create new channel/stream
let id = if initiator: m.currentId.inc(); m.currentId else: chanId
trace "creating new channel", channelId = id, initiator = initiator
result = newChannel(id, m.connection, initiator, name)
m.getChannelList(initiator)[id] = result
proc cleanupChann(m: Mplex, chann: LPChannel, initiator: bool) {.async, inline.} =
## call the channel's `close` to signal the
## remote that the channel is closing
if not isNil(chann) and not chann.closed:
await chann.close()
await chann.cleanUp()
m.getChannelList(initiator).del(chann.id)
trace "cleaned up channel", id = chann.id
method handle*(m: Mplex) {.async, gcsafe.} =
trace "starting mplex main loop"
try:
while not m.connection.closed:
trace "waiting for data"
let msg = await m.connection.readMsg()
if msg.isNone:
# TODO: allow poll with timeout to avoid using `sleepAsync`
await sleepAsync(1.millis)
continue
let (id, msgType, data) = msg.get()
let initiator = bool(ord(msgType) and 1)
var channel: LPChannel
if MessageType(msgType) != MessageType.New:
let channels = m.getChannelList(initiator)
if id notin channels:
trace "Channel not found, skipping", id = id,
initiator = initiator,
msg = msgType
await sleepAsync(1.millis)
continue
channel = channels[id]
case msgType:
of MessageType.New:
let name = cast[string](data)
channel = await m.newStreamInternal(false, id, name)
trace "created channel", id = id, name = name, inititator = true
if not isNil(m.streamHandler):
let stream = newConnection(channel)
stream.peerInfo = m.connection.peerInfo
# cleanup channel once handler is finished
# stream.closeEvent.wait().addCallback(
# proc(udata: pointer) =
# asyncCheck cleanupChann(m, channel, initiator))
asyncCheck m.streamHandler(stream)
continue
of MessageType.MsgIn, MessageType.MsgOut:
trace "pushing data to channel", id = id,
initiator = initiator,
msgType = msgType
await channel.pushTo(data)
of MessageType.CloseIn, MessageType.CloseOut:
trace "closing channel", id = id,
initiator = initiator,
msgType = msgType
await channel.closedByRemote()
# m.getChannelList(initiator).del(id)
of MessageType.ResetIn, MessageType.ResetOut:
trace "resetting channel", id = id,
initiator = initiator,
msgType = msgType
await channel.resetByRemote()
m.getChannelList(initiator).del(id)
break
except CatchableError as exc:
trace "exception occurred", exception = exc.msg
finally:
trace "stopping mplex main loop"
await m.connection.close()
proc newMplex*(conn: Connection,
maxChanns: uint = MaxChannels): Mplex =
new result
result.connection = conn
result.maxChannels = maxChanns
result.remote = initTable[uint, LPChannel]()
result.local = initTable[uint, LPChannel]()
let m = result
conn.closeEvent.wait().addCallback(
proc(udata: pointer) =
trace "connection closed, cleaning up mplex"
asyncCheck m.close()
)
method newStream*(m: Mplex, name: string = ""): Future[Connection] {.async, gcsafe.} =
let channel = await m.newStreamInternal()
# TODO: open the channel (this should be lazy)
await channel.open()
result = newConnection(channel)
result.peerInfo = m.connection.peerInfo
method close*(m: Mplex) {.async, gcsafe.} =
trace "closing mplex muxer"
await allFutures(@[allFutures(toSeq(m.remote.values).mapIt(it.reset())),
allFutures(toSeq(m.local.values).mapIt(it.reset()))])