From a56c3bc296257c43025eb30f767498825f02d4ee Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Thu, 22 Sep 2022 21:55:59 +0200 Subject: [PATCH 01/23] Make observedAddr optional (#772) Co-authored-by: Tanguy --- libp2p/dial.nim | 5 ++- libp2p/dialer.nim | 5 +-- libp2p/protocols/connectivity/autonat.nim | 18 ++++++++--- libp2p/protocols/identify.nim | 6 ++-- libp2p/protocols/pubsub/pubsubpeer.nim | 5 +-- libp2p/protocols/secure/secure.nim | 5 +-- libp2p/stream/chronosstream.nim | 5 ++- libp2p/stream/connection.nim | 9 +++--- libp2p/transports/tcptransport.nim | 21 ++++++++----- libp2p/transports/wstransport.nim | 10 +++--- tests/commontransport.nim | 8 +++-- tests/testconnmngr.nim | 38 +++++++++++++---------- 12 files changed, 83 insertions(+), 52 deletions(-) diff --git a/libp2p/dial.nim b/libp2p/dial.nim index bb8d00cd2..e0d78b906 100644 --- a/libp2p/dial.nim +++ b/libp2p/dial.nim @@ -13,10 +13,13 @@ else: {.push raises: [].} import chronos +import stew/results import peerid, stream/connection, transports/transport +export results + type Dial* = ref object of RootObj @@ -69,5 +72,5 @@ method addTransport*( method tryDial*( self: Dial, peerId: PeerId, - addrs: seq[MultiAddress]): Future[MultiAddress] {.async, base.} = + addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} = doAssert(false, "Not implemented!") diff --git a/libp2p/dialer.nim b/libp2p/dialer.nim index 85c5b63d9..30ff15e27 100644 --- a/libp2p/dialer.nim +++ b/libp2p/dialer.nim @@ -9,6 +9,7 @@ import std/[sugar, tables] +import stew/results import pkg/[chronos, chronicles, metrics] @@ -24,7 +25,7 @@ import dial, upgrademngrs/upgrade, errors -export dial, errors +export dial, errors, results logScope: topics = "libp2p dialer" @@ -189,7 +190,7 @@ proc negotiateStream( method tryDial*( self: Dialer, peerId: PeerId, - addrs: seq[MultiAddress]): Future[MultiAddress] {.async.} = + addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} = ## Create a protocol stream in order to check ## if a connection is possible. ## Doesn't use the Connection Manager to save it. diff --git a/libp2p/protocols/connectivity/autonat.nim b/libp2p/protocols/connectivity/autonat.nim index 4f7fb53eb..6408a7e04 100644 --- a/libp2p/protocols/connectivity/autonat.nim +++ b/libp2p/protocols/connectivity/autonat.nim @@ -13,6 +13,7 @@ else: {.push raises: [].} import std/[options, sets, sequtils] +import stew/results import chronos, chronicles, stew/objects import ../protocol, ../../switch, @@ -226,7 +227,10 @@ proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} = try: await a.sem.acquire() let ma = await a.switch.dialer.tryDial(conn.peerId, addrs) - await conn.sendResponseOk(ma) + if ma.isSome: + await conn.sendResponseOk(ma.get()) + else: + await conn.sendResponseError(DialError, "Missing observed address") except CancelledError as exc: raise exc except CatchableError as exc: @@ -241,15 +245,19 @@ proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] = if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId: return conn.sendResponseError(BadRequest, "PeerId mismatch") - var isRelayed = conn.observedAddr.contains(multiCodec("p2p-circuit")) + if conn.observedAddr.isNone: + return conn.sendResponseError(BadRequest, "Missing observed address") + let observedAddr = conn.observedAddr.get() + + var isRelayed = observedAddr.contains(multiCodec("p2p-circuit")) if isRelayed.isErr() or isRelayed.get(): return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address") - let hostIp = conn.observedAddr[0] + let hostIp = observedAddr[0] if hostIp.isErr() or not IP.match(hostIp.get()): - trace "wrong observed address", address=conn.observedAddr + trace "wrong observed address", address=observedAddr return conn.sendResponseError(InternalError, "Expected an IP address") var addrs = initHashSet[MultiAddress]() - addrs.incl(conn.observedAddr) + addrs.incl(observedAddr) for ma in peerInfo.addrs: isRelayed = ma.contains(multiCodec("p2p-circuit")) if isRelayed.isErr() or isRelayed.get(): diff --git a/libp2p/protocols/identify.nim b/libp2p/protocols/identify.nim index 975126ff8..ae7d26c48 100644 --- a/libp2p/protocols/identify.nim +++ b/libp2p/protocols/identify.nim @@ -16,6 +16,7 @@ else: {.push raises: [].} import std/[sequtils, options, strutils, sugar] +import stew/results import chronos, chronicles import ../protobuf/minprotobuf, ../peerinfo, @@ -80,7 +81,7 @@ chronicles.expandIt(IdentifyInfo): if iinfo.signedPeerRecord.isSome(): "Some" else: "None" -proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): ProtoBuffer +proc encodeMsg(peerInfo: PeerInfo, observedAddr: Opt[MultiAddress], sendSpr: bool): ProtoBuffer {.raises: [Defect].} = result = initProtoBuffer() @@ -91,7 +92,8 @@ proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): P result.write(2, ma.data.buffer) for proto in peerInfo.protocols: result.write(3, proto) - result.write(4, observedAddr.data.buffer) + if observedAddr.isSome: + result.write(4, observedAddr.get().data.buffer) let protoVersion = ProtoVersion result.write(5, protoVersion) let agentVersion = if peerInfo.agentVersion.len <= 0: diff --git a/libp2p/protocols/pubsub/pubsubpeer.nim b/libp2p/protocols/pubsub/pubsubpeer.nim index 8af49de8a..5a1afedbd 100644 --- a/libp2p/protocols/pubsub/pubsubpeer.nim +++ b/libp2p/protocols/pubsub/pubsubpeer.nim @@ -12,7 +12,8 @@ when (NimMajor, NimMinor) < (1, 4): else: {.push raises: [].} -import std/[sequtils, strutils, tables, hashes] +import std/[sequtils, strutils, tables, hashes, options] +import stew/results import chronos, chronicles, nimcrypto/sha2, metrics import rpc/[messages, message, protobuf], ../../peerid, @@ -174,7 +175,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} = trace "Get new send connection", p, newConn p.sendConn = newConn - p.address = some(p.sendConn.observedAddr) + p.address = if p.sendConn.observedAddr.isSome: some(p.sendConn.observedAddr.get) else: none(MultiAddress) if p.onEvent != nil: p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Connected)) diff --git a/libp2p/protocols/secure/secure.nim b/libp2p/protocols/secure/secure.nim index 0bdf85209..c187fd2fb 100644 --- a/libp2p/protocols/secure/secure.nim +++ b/libp2p/protocols/secure/secure.nim @@ -13,6 +13,7 @@ else: {.push raises: [].} import std/[strformat] +import stew/results import chronos, chronicles import ../protocol, ../../stream/streamseq, @@ -21,7 +22,7 @@ import ../protocol, ../../peerinfo, ../../errors -export protocol +export protocol, results logScope: topics = "libp2p secure" @@ -48,7 +49,7 @@ chronicles.formatIt(SecureConn): shortLog(it) proc new*(T: type SecureConn, conn: Connection, peerId: PeerId, - observedAddr: MultiAddress, + observedAddr: Opt[MultiAddress], timeout: Duration = DefaultConnectionTimeout): T = result = T(stream: conn, peerId: peerId, diff --git a/libp2p/stream/chronosstream.nim b/libp2p/stream/chronosstream.nim index 2b676d468..d123c98be 100644 --- a/libp2p/stream/chronosstream.nim +++ b/libp2p/stream/chronosstream.nim @@ -13,10 +13,13 @@ else: {.push raises: [].} import std/[oids, strformat] +import stew/results import chronos, chronicles, metrics import connection import ../utility +export results + logScope: topics = "libp2p chronosstream" @@ -60,7 +63,7 @@ proc init*(C: type ChronosStream, client: StreamTransport, dir: Direction, timeout = DefaultChronosStreamTimeout, - observedAddr: MultiAddress = MultiAddress()): ChronosStream = + observedAddr: Opt[MultiAddress]): ChronosStream = result = C(client: client, timeout: timeout, dir: dir, diff --git a/libp2p/stream/connection.nim b/libp2p/stream/connection.nim index 43f58af66..a4f52deae 100644 --- a/libp2p/stream/connection.nim +++ b/libp2p/stream/connection.nim @@ -13,13 +13,14 @@ else: {.push raises: [].} import std/[hashes, oids, strformat] +import stew/results import chronicles, chronos, metrics import lpstream, ../multiaddress, ../peerinfo, ../errors -export lpstream, peerinfo, errors +export lpstream, peerinfo, errors, results logScope: topics = "libp2p connection" @@ -37,7 +38,7 @@ type timerTaskFut: Future[void] # the current timer instance timeoutHandler*: TimeoutHandler # timeout handler peerId*: PeerId - observedAddr*: MultiAddress + observedAddr*: Opt[MultiAddress] upgraded*: Future[void] protocol*: string # protocol used by the connection, used as tag for metrics transportDir*: Direction # The bottom level transport (generally the socket) direction @@ -160,9 +161,9 @@ method getWrapped*(s: Connection): Connection {.base.} = proc new*(C: type Connection, peerId: PeerId, dir: Direction, + observedAddr: Opt[MultiAddress], timeout: Duration = DefaultConnectionTimeout, - timeoutHandler: TimeoutHandler = nil, - observedAddr: MultiAddress = MultiAddress()): Connection = + timeoutHandler: TimeoutHandler = nil): Connection = result = C(peerId: peerId, dir: dir, timeout: timeout, diff --git a/libp2p/transports/tcptransport.nim b/libp2p/transports/tcptransport.nim index 24f63f6af..6d0d32182 100644 --- a/libp2p/transports/tcptransport.nim +++ b/libp2p/transports/tcptransport.nim @@ -15,6 +15,7 @@ else: {.push raises: [].} import std/[oids, sequtils] +import stew/results import chronos, chronicles import transport, ../errors, @@ -31,7 +32,7 @@ import transport, logScope: topics = "libp2p tcptransport" -export transport +export transport, results const TcpTransportTrackerName* = "libp2p.tcptransport" @@ -71,18 +72,20 @@ proc setupTcpTransportTracker(): TcpTransportTracker = result.isLeaked = leakTransport addTracker(TcpTransportTrackerName, result) -proc connHandler*(self: TcpTransport, - client: StreamTransport, - dir: Direction): Future[Connection] {.async.} = - var observedAddr: MultiAddress = MultiAddress() +proc getObservedAddr(client: StreamTransport): Future[MultiAddress] {.async.} = try: - observedAddr = MultiAddress.init(client.remoteAddress).tryGet() + return MultiAddress.init(client.remoteAddress).tryGet() except CatchableError as exc: trace "Failed to create observedAddr", exc = exc.msg if not(isNil(client) and client.closed): await client.closeWait() raise exc +proc connHandler*(self: TcpTransport, + client: StreamTransport, + observedAddr: Opt[MultiAddress], + dir: Direction): Future[Connection] {.async.} = + trace "Handling tcp connection", address = $observedAddr, dir = $dir, clients = self.clients[Direction.In].len + @@ -222,7 +225,8 @@ method accept*(self: TcpTransport): Future[Connection] {.async, gcsafe.} = self.acceptFuts[index] = self.servers[index].accept() let transp = await finished - return await self.connHandler(transp, Direction.In) + let observedAddr = await getObservedAddr(transp) + return await self.connHandler(transp, Opt.some(observedAddr), Direction.In) except TransportOsError as exc: # TODO: it doesn't sound like all OS errors # can be ignored, we should re-raise those @@ -250,7 +254,8 @@ method dial*( let transp = await connect(address) try: - return await self.connHandler(transp, Direction.Out) + let observedAddr = await getObservedAddr(transp) + return await self.connHandler(transp, Opt.some(observedAddr), Direction.Out) except CatchableError as err: await transp.closeWait() raise err diff --git a/libp2p/transports/wstransport.nim b/libp2p/transports/wstransport.nim index 50eb7c333..9ded1c1e7 100644 --- a/libp2p/transports/wstransport.nim +++ b/libp2p/transports/wstransport.nim @@ -15,6 +15,7 @@ else: {.push raises: [].} import std/[sequtils] +import stew/results import chronos, chronicles import transport, ../errors, @@ -31,7 +32,7 @@ import transport, logScope: topics = "libp2p wstransport" -export transport, websock +export transport, websock, results const WsTransportTrackerName* = "libp2p.wstransport" @@ -45,8 +46,8 @@ type proc new*(T: type WsStream, session: WSSession, dir: Direction, - timeout = 10.minutes, - observedAddr: MultiAddress = MultiAddress()): T = + observedAddr: Opt[MultiAddress], + timeout = 10.minutes): T = let stream = T( session: session, @@ -221,8 +222,7 @@ proc connHandler(self: WsTransport, await stream.close() raise exc - let conn = WsStream.new(stream, dir) - conn.observedAddr = observedAddr + let conn = WsStream.new(stream, dir, Opt.some(observedAddr)) self.connections[dir].add(conn) proc onClose() {.async.} = diff --git a/tests/commontransport.nim b/tests/commontransport.nim index 532689e04..af29add61 100644 --- a/tests/commontransport.nim +++ b/tests/commontransport.nim @@ -1,7 +1,7 @@ {.used.} import sequtils -import chronos, stew/byteutils +import chronos, stew/[byteutils, results] import ../libp2p/[stream/connection, transports/transport, upgrademngrs/upgrade, @@ -35,14 +35,16 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = proc acceptHandler() {.async, gcsafe.} = let conn = await transport1.accept() - check transport1.handles(conn.observedAddr) + if conn.observedAddr.isSome(): + check transport1.handles(conn.observedAddr.get()) await conn.close() let handlerWait = acceptHandler() let conn = await transport2.dial(transport1.addrs[0]) - check transport2.handles(conn.observedAddr) + if conn.observedAddr.isSome(): + check transport2.handles(conn.observedAddr.get()) await conn.close() #for some protocols, closing requires actively reading, so we must close here diff --git a/tests/testconnmngr.nim b/tests/testconnmngr.nim index 5c705bf36..dda2e3dc1 100644 --- a/tests/testconnmngr.nim +++ b/tests/testconnmngr.nim @@ -1,4 +1,5 @@ import sequtils +import stew/results import chronos import ../libp2p/[connmanager, stream/connection, @@ -9,6 +10,9 @@ import ../libp2p/[connmanager, import helpers +proc getConnection(peerId: PeerId, dir: Direction = Direction.In): Connection = + return Connection.new(peerId, dir, Opt.none(MultiAddress)) + type TestMuxer = ref object of Muxer peerId: PeerId @@ -18,7 +22,7 @@ method newStream*( name: string = "", lazy: bool = false): Future[Connection] {.async, gcsafe.} = - result = Connection.new(m.peerId, Direction.Out) + result = getConnection(m.peerId, Direction.Out) suite "Connection Manager": teardown: @@ -27,7 +31,7 @@ suite "Connection Manager": asyncTest "add and retrieve a connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) connMngr.storeConn(conn) check conn in connMngr @@ -41,7 +45,7 @@ suite "Connection Manager": asyncTest "shouldn't allow a closed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) await conn.close() expect CatchableError: @@ -52,7 +56,7 @@ suite "Connection Manager": asyncTest "shouldn't allow an EOFed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) conn.isEof = true expect CatchableError: @@ -64,7 +68,7 @@ suite "Connection Manager": asyncTest "add and retrieve a muxer": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -80,7 +84,7 @@ suite "Connection Manager": asyncTest "shouldn't allow a muxer for an untracked connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -94,8 +98,8 @@ suite "Connection Manager": asyncTest "get conn with direction": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn1 = Connection.new(peerId, Direction.Out) - let conn2 = Connection.new(peerId, Direction.In) + let conn1 = getConnection(peerId, Direction.Out) + let conn2 = getConnection(peerId) connMngr.storeConn(conn1) connMngr.storeConn(conn2) @@ -114,7 +118,7 @@ suite "Connection Manager": asyncTest "get muxed stream for peer": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -134,7 +138,7 @@ suite "Connection Manager": asyncTest "get stream from directed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -155,7 +159,7 @@ suite "Connection Manager": asyncTest "get stream from any connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -175,11 +179,11 @@ suite "Connection Manager": let connMngr = ConnManager.new(maxConnsPerPeer = 1) let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - connMngr.storeConn(Connection.new(peerId, Direction.In)) + connMngr.storeConn(getConnection(peerId)) let conns = @[ - Connection.new(peerId, Direction.In), - Connection.new(peerId, Direction.In)] + getConnection(peerId), + getConnection(peerId)] expect TooManyConnectionsError: connMngr.storeConn(conns[0]) @@ -193,7 +197,7 @@ suite "Connection Manager": asyncTest "cleanup on connection close": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -220,7 +224,7 @@ suite "Connection Manager": Direction.In else: Direction.Out - let conn = Connection.new(peerId, dir) + let conn = getConnection(peerId, dir) let muxer = new Muxer muxer.connection = conn @@ -353,7 +357,7 @@ suite "Connection Manager": let slot = await ((connMngr.getOutgoingSlot()).wait(10.millis)) let conn = - Connection.new( + getConnection( PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet(), Direction.In) From fa5d102370cff4e68b47ac633153b22609b8aa41 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 26 Sep 2022 11:03:24 +0200 Subject: [PATCH 02/23] Better dnsaddr resolving (#753) --- libp2p/dialer.nim | 134 ++++++++++++++++++-------- libp2p/multiaddress.nim | 24 ++++- libp2p/nameresolving/nameresolver.nim | 64 ++++++------ tests/testmultiaddress.nim | 5 + tests/testnameresolve.nim | 17 +++- tests/testnoise.nim | 2 +- tests/testswitch.nim | 23 ++++- 7 files changed, 177 insertions(+), 92 deletions(-) diff --git a/libp2p/dialer.nim b/libp2p/dialer.nim index 30ff15e27..d3c0826dd 100644 --- a/libp2p/dialer.nim +++ b/libp2p/dialer.nim @@ -7,7 +7,7 @@ # This file may not be copied, modified, or distributed except according to # those terms. -import std/[sugar, tables] +import std/[sugar, tables, sequtils] import stew/results import pkg/[chronos, @@ -17,6 +17,7 @@ import pkg/[chronos, import dial, peerid, peerinfo, + multicodec, multistream, connmanager, stream/connection, @@ -46,56 +47,105 @@ type transports: seq[Transport] nameResolver: NameResolver +proc dialAndUpgrade( + self: Dialer, + peerId: Opt[PeerId], + hostname: string, + address: MultiAddress): + Future[Connection] {.async.} = + + for transport in self.transports: # for each transport + if transport.handles(address): # check if it can dial it + trace "Dialing address", address, peerId, hostname + let dialed = + try: + libp2p_total_dial_attempts.inc() + await transport.dial(hostname, address) + except CancelledError as exc: + debug "Dialing canceled", msg = exc.msg, peerId + raise exc + except CatchableError as exc: + debug "Dialing failed", msg = exc.msg, peerId + libp2p_failed_dials.inc() + return nil # Try the next address + + # also keep track of the connection's bottom unsafe transport direction + # required by gossipsub scoring + dialed.transportDir = Direction.Out + + libp2p_successful_dials.inc() + + let conn = + try: + await transport.upgradeOutgoing(dialed, peerId) + except CatchableError as exc: + # If we failed to establish the connection through one transport, + # we won't succeeded through another - no use in trying again + await dialed.close() + debug "Upgrade failed", msg = exc.msg, peerId + if exc isnot CancelledError: + libp2p_failed_upgrades_outgoing.inc() + + # Try other address + return nil + + doAssert not isNil(conn), "connection died after upgradeOutgoing" + debug "Dial successful", conn, peerId = conn.peerId + return conn + return nil + +proc expandDnsAddr( + self: Dialer, + peerId: Opt[PeerId], + address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} = + + if not DNSADDR.matchPartial(address): return @[(address, peerId)] + if isNil(self.nameResolver): + info "Can't resolve DNSADDR without NameResolver", ma=address + return @[] + + let + toResolve = + if peerId.isSome: + address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet() + else: + address + resolved = await self.nameResolver.resolveDnsAddr(toResolve) + + for resolvedAddress in resolved: + let lastPart = resolvedAddress[^1].tryGet() + if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")): + let + peerIdBytes = lastPart.protoArgument().tryGet() + addrPeerId = PeerId.init(peerIdBytes).tryGet() + result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId))) + else: + result.add((resolvedAddress, peerId)) + proc dialAndUpgrade( self: Dialer, peerId: Opt[PeerId], addrs: seq[MultiAddress]): Future[Connection] {.async.} = + debug "Dialing peer", peerId - for address in addrs: # for each address - let - hostname = address.getHostname() - resolvedAddresses = - if isNil(self.nameResolver): @[address] - else: await self.nameResolver.resolveMAddress(address) + for rawAddress in addrs: + # resolve potential dnsaddr + let addresses = await self.expandDnsAddr(peerId, rawAddress) - for a in resolvedAddresses: # for each resolved address - for transport in self.transports: # for each transport - if transport.handles(a): # check if it can dial it - trace "Dialing address", address = $a, peerId, hostname - let dialed = try: - libp2p_total_dial_attempts.inc() - await transport.dial(hostname, a) - except CancelledError as exc: - debug "Dialing canceled", msg = exc.msg, peerId - raise exc - except CatchableError as exc: - debug "Dialing failed", msg = exc.msg, peerId - libp2p_failed_dials.inc() - continue # Try the next address + for (expandedAddress, addrPeerId) in addresses: + # DNS resolution + let + hostname = expandedAddress.getHostname() + resolvedAddresses = + if isNil(self.nameResolver): @[expandedAddress] + else: await self.nameResolver.resolveMAddress(expandedAddress) - # also keep track of the connection's bottom unsafe transport direction - # required by gossipsub scoring - dialed.transportDir = Direction.Out - - libp2p_successful_dials.inc() - - let conn = try: - await transport.upgradeOutgoing(dialed, peerId) - except CatchableError as exc: - # If we failed to establish the connection through one transport, - # we won't succeeded through another - no use in trying again - # TODO we should try another address though - await dialed.close() - debug "Upgrade failed", msg = exc.msg, peerId - if exc isnot CancelledError: - libp2p_failed_upgrades_outgoing.inc() - raise exc - - doAssert not isNil(conn), "connection died after upgradeOutgoing" - debug "Dial successful", conn, peerId = conn.peerId - return conn + for resolvedAddress in resolvedAddresses: + result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress) + if not isNil(result): + return result proc internalConnect( self: Dialer, diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index 054fb00e8..fd663f94a 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -573,7 +573,7 @@ proc protoArgument*(ma: MultiAddress, err("multiaddress: Decoding protocol error") else: ok(res) - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(buffer) == -1: err("multiaddress: Decoding protocol error") else: @@ -594,6 +594,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] = buffer.setLen(res) ok(buffer) +proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] = + ## Returns MultiAddress ``ma`` protocol address binary blob. + ## + ## If current MultiAddress do not have argument value, then result array will + ## be empty. + ma.protoAddress() + proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = var header: uint64 var data = newSeq[byte]() @@ -601,6 +608,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = var vb = ma var res: MultiAddress res.data = initVBuffer() + + if index < 0: return err("multiaddress: negative index gived to getPart") + while offset <= index: if vb.data.readVarint(header) == -1: return err("multiaddress: Malformed binary address!") @@ -618,7 +628,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = res.data.writeVarint(header) res.data.writeArray(data) res.data.finish() - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(data) == -1: return err("multiaddress: Decoding protocol error") @@ -647,9 +657,13 @@ proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddres ? res.append(? ma[i]) ok(res) -proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} = +proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} = ## Returns part with index ``i`` of MultiAddress ``ma``. - ma.getPart(i) + when i is BackwardsIndex: + let maLength = ? len(ma) + ma.getPart(maLength - int(i)) + else: + ma.getPart(i) proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} = ## Returns parts with slice ``slice`` of MultiAddress ``ma``. @@ -680,7 +694,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] = res.data.writeVarint(header) res.data.writeArray(data) - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(data) == -1: yield err(MaResult[MultiAddress], "Decoding protocol error") diff --git a/libp2p/nameresolving/nameresolver.nim b/libp2p/nameresolving/nameresolver.nim index 0fdd6a029..b53e0b4ed 100644 --- a/libp2p/nameresolving/nameresolver.nim +++ b/libp2p/nameresolving/nameresolver.nim @@ -13,7 +13,7 @@ else: {.push raises: [].} import std/[sugar, sets, sequtils, strutils] -import +import chronos, chronicles, stew/[endians2, byteutils] @@ -22,14 +22,14 @@ import ".."/[multiaddress, multicodec] logScope: topics = "libp2p nameresolver" -type +type NameResolver* = ref object of RootObj method resolveTxt*( self: NameResolver, address: string): Future[seq[string]] {.async, base.} = ## Get TXT record - ## + ## doAssert(false, "Not implemented!") @@ -39,16 +39,18 @@ method resolveIp*( port: Port, domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async, base.} = ## Resolve the specified address - ## + ## doAssert(false, "Not implemented!") proc getHostname*(ma: MultiAddress): string = - let firstPart = ($ma[0].get()).split('/') - if firstPart.len > 1: firstPart[2] + let + firstPart = ma[0].valueOr: return "" + fpSplitted = ($firstPart).split('/', 2) + if fpSplitted.len > 2: fpSplitted[2] else: "" -proc resolveDnsAddress( +proc resolveOneAddress( self: NameResolver, ma: MultiAddress, domain: Domain = Domain.AF_UNSPEC, @@ -64,29 +66,22 @@ proc resolveDnsAddress( let port = Port(fromBytesBE(uint16, pbuf)) resolvedAddresses = await self.resolveIp(prefix & dnsval, port, domain) - + return collect(newSeqOfCap(4)): for address in resolvedAddresses: var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet() for part in ma: - if DNS.match(part.get()): continue + if DNS.match(part.tryGet()): continue createdAddress &= part.tryGet() createdAddress -func matchDnsSuffix(m1, m2: MultiAddress): MaResult[bool] = - for partMaybe in m1: - let part = ?partMaybe - if DNS.match(part): continue - let entryProt = ?m2[?part.protoCode()] - if entryProt != part: - return ok(false) - return ok(true) - -proc resolveDnsAddr( +proc resolveDnsAddr*( self: NameResolver, ma: MultiAddress, - depth: int = 0): Future[seq[MultiAddress]] - {.async.} = + depth: int = 0): Future[seq[MultiAddress]] {.async.} = + + if not DNSADDR.matchPartial(ma): + return @[ma] trace "Resolving dnsaddr", ma if depth > 6: @@ -104,21 +99,17 @@ proc resolveDnsAddr( if not entry.startsWith("dnsaddr="): continue let entryValue = MultiAddress.init(entry[8..^1]).tryGet() - if not matchDnsSuffix(ma, entryValue).tryGet(): continue + if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet(): + if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]: + continue - # The spec is not clear wheter only DNSADDR can be recursived - # or any DNS addr. Only handling DNSADDR because it's simpler - # to avoid infinite recursion - if DNSADDR.matchPartial(entryValue): - let resolved = await self.resolveDnsAddr(entryValue, depth + 1) - for r in resolved: - result.add(r) - else: - result.add(entryValue) + let resolved = await self.resolveDnsAddr(entryValue, depth + 1) + for r in resolved: + result.add(r) if result.len == 0: - debug "Failed to resolve any DNSADDR", ma - return @[ma] + debug "Failed to resolve a DNSADDR", ma + return @[] return result @@ -133,14 +124,15 @@ proc resolveMAddress*( let code = address[0].get().protoCode().get() let seq = case code: of multiCodec("dns"): - await self.resolveDnsAddress(address) + await self.resolveOneAddress(address) of multiCodec("dns4"): - await self.resolveDnsAddress(address, Domain.AF_INET) + await self.resolveOneAddress(address, Domain.AF_INET) of multiCodec("dns6"): - await self.resolveDnsAddress(address, Domain.AF_INET6) + await self.resolveOneAddress(address, Domain.AF_INET6) of multiCodec("dnsaddr"): await self.resolveDnsAddr(address) else: + doAssert false @[address] for ad in seq: res.incl(ad) diff --git a/tests/testmultiaddress.nim b/tests/testmultiaddress.nim index 2c05e212e..c180db7f2 100644 --- a/tests/testmultiaddress.nim +++ b/tests/testmultiaddress.nim @@ -386,6 +386,11 @@ suite "MultiAddress test suite": let ma = MultiAddress.init("/ip4/0.0.0.0/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio/").get() check: $ma[0..0].get() == "/ip4/0.0.0.0" + $ma[^1].get() == "/unix/stdio" + ma[-100].isErr() + ma[100].isErr() + ma[^100].isErr() + ma[^0].isErr() $ma[0..1].get() == "/ip4/0.0.0.0/tcp/0" $ma[1..2].get() == "/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" $ma[^3..^1].get() == "/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio" diff --git a/tests/testnameresolve.nim b/tests/testnameresolve.nim index fa9b88ffa..4a4dd40b0 100644 --- a/tests/testnameresolve.nim +++ b/tests/testnameresolve.nim @@ -139,7 +139,18 @@ suite "Name resolving": asyncTest "dnsaddr infinite recursion": resolver.txtResponses["_dnsaddr.bootstrap.libp2p.io"] = @["dnsaddr=/dnsaddr/bootstrap.libp2p.io"] - check testOne("/dnsaddr/bootstrap.libp2p.io/", "/dnsaddr/bootstrap.libp2p.io/") + check testOne("/dnsaddr/bootstrap.libp2p.io/", newSeq[string]()) + + test "getHostname": + check: + MultiAddress.init("/dnsaddr/bootstrap.libp2p.io/").tryGet().getHostname == "bootstrap.libp2p.io" + MultiAddress.init("").tryGet().getHostname == "" + MultiAddress.init("/ip4/147.75.69.143/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "147.75.69.143" + MultiAddress.init("/ip6/2604:1380:1000:6000::1/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "2604:1380:1000:6000::1" + MultiAddress.init("/dns/localhost/udp/0").tryGet().getHostname == "localhost" + MultiAddress.init("/dns4/hello.com/udp/0").tryGet().getHostname == "hello.com" + MultiAddress.init("/dns6/hello.com/udp/0").tryGet().getHostname == "hello.com" + MultiAddress.init("/wss/").tryGet().getHostname == "" suite "DNS Resolving": teardown: @@ -171,7 +182,7 @@ suite "Name resolving": # The test var dnsresolver = DnsResolver.new(@[server.localAddress]) - + check await(dnsresolver.resolveIp("status.im", 0.Port, Domain.AF_UNSPEC)) == mapIt( @["104.22.24.181:0", "172.67.10.161:0", "104.22.25.181:0", @@ -209,7 +220,7 @@ suite "Name resolving": # The test var dnsresolver = DnsResolver.new(@[unresponsiveServer.localAddress, server.localAddress]) - + check await(dnsresolver.resolveIp("status.im", 0.Port, Domain.AF_INET)) == mapIt(@["104.22.24.181:0", "172.67.10.161:0", "104.22.25.181:0"], initTAddress(it)) diff --git a/tests/testnoise.nim b/tests/testnoise.nim index b0e785be3..a94714ce0 100644 --- a/tests/testnoise.nim +++ b/tests/testnoise.nim @@ -295,7 +295,7 @@ suite "Noise": (switch2, peerInfo2) = createSwitch(ma2, true, true) # secio, we want to fail await switch1.start() await switch2.start() - expect(UpgradeFailedError): + expect(DialFailedError): let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) await allFuturesThrowing( diff --git a/tests/testswitch.nim b/tests/testswitch.nim index 608266c09..961b6dc3c 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -201,13 +201,25 @@ suite "Switch": check not switch1.isConnected(switch2.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - asyncTest "e2e connect to peer with unkown PeerId": + asyncTest "e2e connect to peer with unknown PeerId": + let resolver = MockResolver.new() let switch1 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) - let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) + let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise], nameResolver = resolver) await switch1.start() await switch2.start() + # via dnsaddr + resolver.txtResponses["_dnsaddr.test.io"] = @[ + "dnsaddr=" & $switch1.peerInfo.addrs[0] & "/p2p/" & $switch1.peerInfo.peerId, + ] + + check: (await switch2.connect(@[MultiAddress.init("/dnsaddr/test.io/").tryGet()])) == switch1.peerInfo.peerId + await switch2.disconnect(switch1.peerInfo.peerId) + + # via direct ip + check not switch2.isConnected(switch1.peerInfo.peerId) check: (await switch2.connect(switch1.peerInfo.addrs)) == switch1.peerInfo.peerId + await switch2.disconnect(switch1.peerInfo.peerId) await allFuturesThrowing( @@ -665,7 +677,7 @@ suite "Switch": await switch.start() var peerId = PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).get() - expect LPStreamClosedError, LPStreamEOFError: + expect DialFailedError: await switch.connect(peerId, transport.addrs) await handlerWait @@ -994,9 +1006,10 @@ suite "Switch": await srcWsSwitch.start() resolver.txtResponses["_dnsaddr.test.io"] = @[ - "dnsaddr=" & $destSwitch.peerInfo.addrs[0], - "dnsaddr=" & $destSwitch.peerInfo.addrs[1] + "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[0][1..^1].tryGet() & "/p2p/" & $destSwitch.peerInfo.peerId, + "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[1][1..^1].tryGet() ] + resolver.ipResponses[("localhost", false)] = @["127.0.0.1"] let testAddr = MultiAddress.init("/dnsaddr/test.io/").tryGet() From 103e199bc0fcce0f7d7eea96c36f773615830d26 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 26 Sep 2022 11:48:03 +0200 Subject: [PATCH 03/23] add `lodestar` to known lib p2p agents documentation (#766) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 61018ee81..e8ec029d8 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ nim c -d:libp2p_expensive_metrics some_file.nim **use identify metrics** ```bash -nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim +nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku some_file.nim ``` **specify gossipsub specific topics to measure** From eb786607023e0773f0d2919d68716034a394b052 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 28 Sep 2022 10:40:53 +0200 Subject: [PATCH 04/23] Docs rework (#776) --- .github/workflows/doc.yml | 8 ++- .gitignore | 1 + examples/README.md | 2 +- examples/circuitrelay.nim | 12 +++- examples/directchat.nim | 2 +- examples/helloworld.nim | 2 +- examples/tutorial_1_connect.md | 108 ---------------------------- examples/tutorial_1_connect.nim | 95 ++++++++++++++++++++++++ examples/tutorial_2_customproto.md | 82 --------------------- examples/tutorial_2_customproto.nim | 73 +++++++++++++++++++ libp2p.nimble | 18 +++-- mkdocs.yml | 9 ++- tools/markdown_builder.nim | 29 ++++++++ 13 files changed, 235 insertions(+), 206 deletions(-) delete mode 100644 examples/tutorial_1_connect.md create mode 100644 examples/tutorial_1_connect.nim delete mode 100644 examples/tutorial_2_customproto.md create mode 100644 examples/tutorial_2_customproto.nim create mode 100644 tools/markdown_builder.nim diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 862c571ea..dce867672 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -63,7 +63,7 @@ jobs: git push origin gh-pages update_site: - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs' name: 'Rebuild website' runs-on: ubuntu-latest steps: @@ -74,8 +74,12 @@ jobs: with: python-version: 3.x + - uses: jiro4989/setup-nim-action@v1 + with: + nim-version: 'stable' + - name: Generate website - run: pip install mkdocs-material && mkdocs build + run: pip install mkdocs-material && nimble website - name: Clone the gh-pages branch uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index ec585b0d3..f93a08d29 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,6 @@ build/ .vscode/ .DS_Store tests/pubsub/testgossipsub +examples/*.md nimble.develop nimble.paths diff --git a/examples/README.md b/examples/README.md index 18b6ca855..07600735e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,5 +2,5 @@ Welcome to the nim-libp2p documentation! -Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and +Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html). diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index b7c66d2ed..bf90d97ba 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -1,6 +1,14 @@ +## # Circuit Relay example +## +## Circuit Relay can be used when a node cannot reach another node +## directly, but can reach it through a another node (the Relay). +## +## That may happen because of NAT, Firewalls, or incompatible transports. +## +## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/). import chronos, stew/byteutils -import ../libp2p, - ../libp2p/protocols/connectivity/relay/[relay, client] +import libp2p, + libp2p/protocols/connectivity/relay/[relay, client] # Helper to create a circuit relay node proc createCircuitRelaySwitch(r: Relay): Switch = diff --git a/examples/directchat.nim b/examples/directchat.nim index 9e7d99ce1..b550d4805 100644 --- a/examples/directchat.nim +++ b/examples/directchat.nim @@ -5,7 +5,7 @@ import strformat, strutils, stew/byteutils, chronos, - ../libp2p + libp2p const DefaultAddr = "/ip4/127.0.0.1/tcp/0" diff --git a/examples/helloworld.nim b/examples/helloworld.nim index 9f5f1cbe7..7f144aa76 100644 --- a/examples/helloworld.nim +++ b/examples/helloworld.nim @@ -1,6 +1,6 @@ import chronos # an efficient library for async import stew/byteutils # various utils -import ../libp2p # when installed through nimble, just use `import libp2p` +import libp2p ## # Create our custom protocol diff --git a/examples/tutorial_1_connect.md b/examples/tutorial_1_connect.md deleted file mode 100644 index 0dd6543d6..000000000 --- a/examples/tutorial_1_connect.md +++ /dev/null @@ -1,108 +0,0 @@ -# Simple ping tutorial - -Hi all, welcome to the first nim-libp2p tutorial! - -!!! tips "" - This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. - -To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. - -Hope you'll find it helpful in your journey of learning. Happy coding! ;) - -## Before you start -The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). - -Install Nim via their [official website](https://nim-lang.org/install.html). -Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. - -You can now install the latest version of `nim-libp2p`: -```bash -nimble install libp2p@#master -``` - -## A simple ping application -We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. - -!!! tips "" - You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder! - -Let's create a `part1.nim`, and import our dependencies: -```nim -import chronos - -import libp2p -import libp2p/protocols/ping -``` -[chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` - -Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: -```nim -proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = - var switch = SwitchBuilder - .new() - .withRng(rng) # Give the application RNG - .withAddress(ma) # Our local address(es) - .withTcpTransport() # Use TCP as transport - .withMplex() # Use Mplex as muxer - .withNoise() # Use Noise as secure manager - .build() - - return switch -``` -This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. - -You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. - - -Let's now start to create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() - pingProtocol = Ping.new(rng=rng) -``` -We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. -The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". - -`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. - -We can now create our two switches: -```nim - let - switch1 = createSwitch(localAddress, rng) - switch2 = createSwitch(localAddress, rng) - - switch1.mount(pingProtocol) - - await switch1.start() - await switch2.start() -``` -We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. - -Now that we've started the nodes, they are listening for incoming peers. -We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. - -We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: -```nim - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) -``` -We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: -```nim - # ping the other node and echo the ping duration - echo "ping: ", await pingProtocol.ping(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() -``` - -And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: -```nim - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports - -waitFor(main()) -``` - -You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. - -In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_1_connect.nim b/examples/tutorial_1_connect.nim new file mode 100644 index 000000000..d8d8c2b5a --- /dev/null +++ b/examples/tutorial_1_connect.nim @@ -0,0 +1,95 @@ +## # Simple ping tutorial +## +## Hi all, welcome to the first nim-libp2p tutorial! +## +## !!! tips "" +## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. +## +## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. +## +## Hope you'll find it helpful in your journey of learning. Happy coding! ;) +## +## ## Before you start +## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). +## +## Install Nim via their [official website](https://nim-lang.org/install.html). +## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. +## +## You can now install the latest version of `nim-libp2p`: +## ```bash +## nimble install libp2p@#master +## ``` +## +## ## A simple ping application +## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. +## +## !!! tips "" +## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder! +## +## Let's create a `part1.nim`, and import our dependencies: +import chronos + +import libp2p +import libp2p/protocols/ping + +## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` +## +## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: +proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = + var switch = SwitchBuilder + .new() + .withRng(rng) # Give the application RNG + .withAddress(ma) # Our local address(es) + .withTcpTransport() # Use TCP as transport + .withMplex() # Use Mplex as muxer + .withNoise() # Use Noise as secure manager + .build() + + return switch + +## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. +## +## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. +## +## +## Let's now start to create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() + pingProtocol = Ping.new(rng=rng) + ## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. + ## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". + ## + ## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. + ## + ## We can now create our two switches: + let + switch1 = createSwitch(localAddress, rng) + switch2 = createSwitch(localAddress, rng) + + switch1.mount(pingProtocol) + + await switch1.start() + await switch2.start() + ## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. + ## + ## Now that we've started the nodes, they are listening for incoming peers. + ## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. + ## + ## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) + ## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: + # ping the other node and echo the ping duration + echo "ping: ", await pingProtocol.ping(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + ## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +waitFor(main()) + +## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. +## +## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_2_customproto.md b/examples/tutorial_2_customproto.md deleted file mode 100644 index aa3366c8b..000000000 --- a/examples/tutorial_2_customproto.md +++ /dev/null @@ -1,82 +0,0 @@ -# Custom protocol in libp2p - -In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. - -We'll now look at how to create a custom protocol inside the libp2p - -Let's create a `part2.nim`, and import our dependencies: -```nim -import chronos -import stew/byteutils - -import libp2p -``` -This is similar to the first tutorial, except we don't need to import the `Ping` protocol. - -Next, we'll declare our custom protocol -```nim -const TestCodec = "/test/proto/1.0.0" - -type TestProto = ref object of LPProtocol -``` - -We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. - -A protocol generally has two part: and handling/server part, and a dialing/client part. -Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. - -Let's start with the server part: -```nim -proc new(T: typedesc[TestProto]): T = - # every incoming connections will in be handled in this closure - proc handle(conn: Connection, proto: string) {.async, gcsafe.} = - # Read up to 1024 bytes from this connection, and transform them into - # a string - echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) - # We must close the connections ourselves when we're done with it - await conn.close() - - return T(codecs: @[TestCodec], handler: handle) -``` -This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. -In our handle, we simply read a message from the connection and `echo` it. - -We can now create our client part: -```nim -proc hello(p: TestProto, conn: Connection) {.async.} = - await conn.writeLp("Hello p2p!") -``` -Again, pretty straight-forward, we just send a message on the connection. - -We can now create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - testProto = TestProto.new() - switch1 = newStandardSwitch(rng=rng) - switch2 = newStandardSwitch(rng=rng) - - switch1.mount(testProto) - - await switch1.start() - await switch2.start() - - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) - - await testProto.hello(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() - - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports -``` - -This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p - -We can now wrap our program by calling our main proc: -```nim -waitFor(main()) -``` - -And that's it! diff --git a/examples/tutorial_2_customproto.nim b/examples/tutorial_2_customproto.nim new file mode 100644 index 000000000..2f5789b16 --- /dev/null +++ b/examples/tutorial_2_customproto.nim @@ -0,0 +1,73 @@ +## # Custom protocol in libp2p +## +## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. +## +## We'll now look at how to create a custom protocol inside the libp2p +## +## Let's create a `part2.nim`, and import our dependencies: +import chronos +import stew/byteutils + +import libp2p +## This is similar to the first tutorial, except we don't need to import the `Ping` protocol. +## +## Next, we'll declare our custom protocol +const TestCodec = "/test/proto/1.0.0" + +type TestProto = ref object of LPProtocol + +## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. +## +## A protocol generally has two part: and handling/server part, and a dialing/client part. +## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. +## +## Let's start with the server part: + +proc new(T: typedesc[TestProto]): T = + # every incoming connections will in be handled in this closure + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + # Read up to 1024 bytes from this connection, and transform them into + # a string + echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) + # We must close the connections ourselves when we're done with it + await conn.close() + + return T(codecs: @[TestCodec], handler: handle) + +## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. +## In our handle, we simply read a message from the connection and `echo` it. +## +## We can now create our client part: +proc hello(p: TestProto, conn: Connection) {.async.} = + await conn.writeLp("Hello p2p!") + +## Again, pretty straight-forward, we just send a message on the connection. +## +## We can now create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + testProto = TestProto.new() + switch1 = newStandardSwitch(rng=rng) + switch2 = newStandardSwitch(rng=rng) + + switch1.mount(testProto) + + await switch1.start() + await switch2.start() + + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) + + await testProto.hello(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p +## +## We can now wrap our program by calling our main proc: +waitFor(main()) + +## And that's it! diff --git a/libp2p.nimble b/libp2p.nimble index fb2ef486c..a1c3b04a0 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -32,16 +32,16 @@ proc runTest(filename: string, verify: bool = true, sign: bool = true, rmFile "tests/" & filename.toExe proc buildSample(filename: string, run = false) = - var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off " + var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " excstr.add(" examples/" & filename) exec excstr if run: exec "./examples/" & filename.toExe rmFile "examples/" & filename.toExe -proc buildTutorial(filename: string) = - discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " & - " nim --verbosity:0 --hints:off c -" +proc tutorialToMd(filename: string) = + let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim " + writeFile(filename.replace(".nim", ".md"), markdown) task testnative, "Runs libp2p native tests": runTest("testnative") @@ -86,12 +86,18 @@ task test_slim, "Runs the (slimmed down) test suite": exec "nimble testfilter" exec "nimble examples_build" +task website, "Build the website": + tutorialToMd("examples/tutorial_1_connect.nim") + tutorialToMd("examples/tutorial_2_customproto.nim") + tutorialToMd("examples/circuitrelay.nim") + exec "mkdocs build" + task examples_build, "Build the samples": buildSample("directchat") buildSample("helloworld", true) buildSample("circuitrelay", true) - buildTutorial("examples/tutorial_1_connect.md") - buildTutorial("examples/tutorial_2_customproto.md") + buildSample("tutorial_1_connect", true) + buildSample("tutorial_2_customproto", true) # pin system # while nimble lockfile diff --git a/mkdocs.yml b/mkdocs.yml index 9c4fcbb29..76c657038 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,9 @@ site_name: nim-libp2p repo_url: https://github.com/status-im/nim-libp2p repo_name: status-im/nim-libp2p site_url: https://status-im.github.io/nim-libp2p/docs -edit_uri: edit/unstable/examples/ +# Can't find a way to point the edit to the .nim instead +# of the .md +edit_uri: '' docs_dir: examples @@ -40,6 +42,7 @@ theme: nav: - Introduction: README.md - Tutorials: - - 'Part I: Simple connection': tutorial_1_connect.md - - 'Part II: Custom protocol': tutorial_2_customproto.md + - 'Simple connection': tutorial_1_connect.md + - 'Create a custom protocol': tutorial_2_customproto.md + - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' diff --git a/tools/markdown_builder.nim b/tools/markdown_builder.nim new file mode 100644 index 000000000..1477c8028 --- /dev/null +++ b/tools/markdown_builder.nim @@ -0,0 +1,29 @@ +import os, strutils + +let contents = + if paramCount() > 0: + readFile(paramStr(1)) + else: + stdin.readAll() + +var code = "" +for line in contents.splitLines(true): + let + stripped = line.strip() + isMarkdown = stripped.startsWith("##") + + if isMarkdown: + if code.strip.len > 0: + echo "```nim" + echo code.strip(leading = false) + echo "```" + code = "" + echo(if stripped.len > 3: stripped[3..^1] + else: "") + else: + code &= line +if code.strip.len > 0: + echo "" + echo "```nim" + echo code + echo "```" From bcb8f5e3b6d9349bf0a10c9a5efd2d9a6467b48f Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 29 Sep 2022 10:28:58 +0200 Subject: [PATCH 05/23] Protobuf tutorial (#778) --- examples/tutorial_2_customproto.nim | 1 + examples/tutorial_3_protobuf.nim | 162 ++++++++++++++++++++++++++++ libp2p.nimble | 4 + mkdocs.yml | 1 + 4 files changed, 168 insertions(+) create mode 100644 examples/tutorial_3_protobuf.nim diff --git a/examples/tutorial_2_customproto.nim b/examples/tutorial_2_customproto.nim index 2f5789b16..be418a7a1 100644 --- a/examples/tutorial_2_customproto.nim +++ b/examples/tutorial_2_customproto.nim @@ -71,3 +71,4 @@ proc main() {.async, gcsafe.} = waitFor(main()) ## And that's it! +## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf. diff --git a/examples/tutorial_3_protobuf.nim b/examples/tutorial_3_protobuf.nim new file mode 100644 index 000000000..2af7efe61 --- /dev/null +++ b/examples/tutorial_3_protobuf.nim @@ -0,0 +1,162 @@ +## # Protobuf usage +## +## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol. +## Most real protocol want their messages to be structured and extensible, which is why +## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to +## define their message structures. +## +## Here, we'll create a slightly more complex protocol, which parses & generate protobuf +## messages. Let's start by importing our dependencies, as usual: +import chronos +import stew/results # for Opt[T] + +import libp2p + +## ## Protobuf encoding & decoding +## This will be the structure of our messages: +## ```protobuf +## message MetricList { +## message Metric { +## string name = 1; +## float value = 2; +## } +## +## repeated Metric metrics = 2; +## } +## ``` +## We'll create our protobuf types, encoders & decoders, according to this format. +## To create the encoders & decoders, we are going to use minprotobuf +## (included in libp2p). +## +## While more modern technics +## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization)) +## exists, minprotobuf is currently the recommended method to handle protobuf, since it has +## been used in production extensively, and audited. +type + Metric = object + name: string + value: float + + MetricList = object + metrics: seq[Metric] + +{.push raises: [].} + +proc encode(m: Metric): ProtoBuffer = + result = initProtoBuffer() + result.write(1, m.name) + result.write(2, m.value) + result.finish() + +proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] = + var res: Metric + let pb = initProtoBuffer(buf) + # "getField" will return a Result[bool, ProtoError]. + # The Result will hold an error if the protobuf is invalid. + # The Result will hold "false" if the field is missing + # + # We are just checking the error, and ignoring whether the value + # is present or not (default values are valid). + discard ? pb.getField(1, res.name) + discard ? pb.getField(2, res.value) + ok(res) + +proc encode(m: MetricList): ProtoBuffer = + result = initProtoBuffer() + for metric in m.metrics: + result.write(1, metric.encode()) + result.finish() + +proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] = + var + res: MetricList + metrics: seq[seq[byte]] + let pb = initProtoBuffer(buf) + discard ? pb.getRepeatedField(1, metrics) + + for metric in metrics: + res.metrics &= ? Metric.decode(metric) + ok(res) + +## ## Results instead of exceptions +## As you can see, this part of the program also uses Results instead of exceptions for error handling. +## We start by `{.push raises: [].}`, which will prevent every non-async function from raising +## exceptions. +## +## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey +## errors to function callers. A `Result[T, E]` will either hold a valid result of type +## T, or an error of type E. +## +## You can check if the call succeeded by using `res.isOk`, and then get the +## value using `res.value` or the error by using `res.error`. +## +## Another useful tool is `?`, which will unpack a Result if it succeeded, +## or if it failed, exit the current procedure returning the error. +## +## nim-result is packed with other functionalities that you'll find in the +## nim-result repository. +## +## Results and exception are generally interchangeable, but have different semantics +## that you may or may not prefer. +## +## ## Creating the protocol +## We'll next create a protocol, like in the last tutorial, to request these metrics from our host +type + MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.} + MetricProto = ref object of LPProtocol + metricGetter: MetricCallback + +proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto = + let res = MetricProto(metricGetter: cb) + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + let + metrics = await res.metricGetter() + asProtobuf = metrics.encode() + await conn.writeLp(asProtobuf.buffer) + await conn.close() + + res.codecs = @["/metric-getter/1.0.0"] + res.handler = handle + return res + +proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} = + let protobuf = await conn.readLp(2048) + # tryGet will raise an exception if the Result contains an error. + # It's useful to bridge between exception-world and result-world + return MetricList.decode(protobuf).tryGet() + +## We can now create our main procedure: +proc main() {.async, gcsafe.} = + let rng = newRng() + proc randomMetricGenerator: Future[MetricList] {.async.} = + let metricCount = rng[].generate(uint32) mod 16 + for i in 0 ..< metricCount + 1: + result.metrics.add(Metric( + name: "metric_" & $i, + value: float(rng[].generate(uint16)) / 1000.0 + )) + return result + let + metricProto1 = MetricProto.new(randomMetricGenerator) + metricProto2 = MetricProto.new(randomMetricGenerator) + switch1 = newStandardSwitch(rng=rng) + switch2 = newStandardSwitch(rng=rng) + + switch1.mount(metricProto1) + + await switch1.start() + await switch2.start() + + let + conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs) + metrics = await metricProto2.fetch(conn) + await conn.close() + + for metric in metrics.metrics: + echo metric.name, " = ", metric.value + + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +waitFor(main()) + +## If you run this program, you should see random metrics being sent from the switch1 to the switch2. diff --git a/libp2p.nimble b/libp2p.nimble index a1c3b04a0..bc83ef8eb 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -89,6 +89,7 @@ task test_slim, "Runs the (slimmed down) test suite": task website, "Build the website": tutorialToMd("examples/tutorial_1_connect.nim") tutorialToMd("examples/tutorial_2_customproto.nim") + tutorialToMd("examples/tutorial_3_protobuf.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -98,6 +99,9 @@ task examples_build, "Build the samples": buildSample("circuitrelay", true) buildSample("tutorial_1_connect", true) buildSample("tutorial_2_customproto", true) + if (NimMajor, NimMinor) > (1, 2): + # This tutorial relies on post 1.4 exception tracking + buildSample("tutorial_3_protobuf", true) # pin system # while nimble lockfile diff --git a/mkdocs.yml b/mkdocs.yml index 76c657038..57d847994 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,5 +44,6 @@ nav: - Tutorials: - 'Simple connection': tutorial_1_connect.md - 'Create a custom protocol': tutorial_2_customproto.md + - 'Protobuf': tutorial_3_protobuf.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' From 0cd3554ce471fe53211c881badeabeaec46f4bd8 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 29 Sep 2022 10:29:51 +0200 Subject: [PATCH 06/23] Bump deps (#779) --- .pinned | 18 +++++++++--------- libp2p.nimble | 2 +- libp2p/nameresolving/dnsresolver.nim | 24 ++++++++++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.pinned b/.pinned index f85d7a2e5..3397fbbad 100644 --- a/.pinned +++ b/.pinned @@ -1,16 +1,16 @@ -bearssl;https://github.com/status-im/nim-bearssl@#25009951ff8e0006171d566e3c7dc73a8231c2ed +bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a -chronos;https://github.com/status-im/nim-chronos@#41b82cdea34744148600b67a9154331b76181189 -dnsclient;https://github.com/ba0f3/dnsclient.nim@#4960de2b345f567b12f09a08e9967af104ab39a3 -faststreams;https://github.com/status-im/nim-faststreams@#49e2c52eb5dda46b1c9c10d079abe7bffe6cea89 -httputils;https://github.com/status-im/nim-http-utils@#f83fbce4d6ec7927b75be3f85e4fa905fcb69788 +chronos;https://github.com/status-im/nim-chronos@#9df76c39df254c7ff0cec6dec5c9f345f2819c91 +dnsclient;https://github.com/ba0f3/dnsclient.nim@#6647ca8bd9ffcc13adaecb9cb6453032063967db +faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29 +httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040 json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 -metrics;https://github.com/status-im/nim-metrics@#9070af9c830e93e5239ddc488cd65aa6f609ba73 +metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178 -secp256k1;https://github.com/status-im/nim-secp256k1@#5340cf188168d6afcafc8023770d880f067c0b2f +secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466 -stew;https://github.com/status-im/nim-stew@#598246620da5c41d0e92a8dd6aab0755381b21cd +stew;https://github.com/status-im/nim-stew@#f2e58ba4c8da65548c824e4fa8732db9739f6505 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34 unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db -websock;https://github.com/status-im/nim-websock@#8a72c0f7690802753b1d59887745b1ce1f0c8b3d +websock;https://github.com/status-im/nim-websock@#2424f2b215c0546f97d8b147e21544521c7545b0 zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8 \ No newline at end of file diff --git a/libp2p.nimble b/libp2p.nimble index bc83ef8eb..eb661d84e 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -9,7 +9,7 @@ skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"] requires "nim >= 1.2.0", "nimcrypto >= 0.4.1", - "dnsclient >= 0.1.2", + "dnsclient >= 0.3.0 & < 0.4.0", "bearssl >= 0.1.4", "chronicles >= 0.10.2", "chronos >= 3.0.6", diff --git a/libp2p/nameresolving/dnsresolver.nim b/libp2p/nameresolving/dnsresolver.nim index 51be45624..a638b2545 100644 --- a/libp2p/nameresolving/dnsresolver.nim +++ b/libp2p/nameresolving/dnsresolver.nim @@ -14,7 +14,7 @@ else: import std/[streams, strutils, sets, sequtils], - chronos, chronicles, + chronos, chronicles, stew/byteutils, dnsclientpkg/[protocol, types] import @@ -76,15 +76,11 @@ proc getDnsResponse( if not receivedDataFuture.finished: raise newException(IOError, "DNS server timeout") - var - rawResponse = sock.getMessage() - dataStream = newStringStream() - dataStream.writeData(addr rawResponse[0], rawResponse.len) - dataStream.setPosition(0) + let rawResponse = sock.getMessage() # parseResponse can has a raises: [Exception, ..] because of # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 # it can't actually raise though - return parseResponse(dataStream) + return parseResponse(string.fromBytes(rawResponse)) except CatchableError as exc: raise exc except Exception as exc: raiseAssert exc.msg finally: @@ -118,7 +114,14 @@ method resolveIp*( try: let resp = await fut for answer in resp.answers: - resolvedAddresses.incl(answer.toString()) + # toString can has a raises: [Exception, ..] because of + # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 + # it can't actually raise though + resolvedAddresses.incl( + try: answer.toString() + except CatchableError as exc: raise exc + except Exception as exc: raiseAssert exc.msg + ) except CancelledError as e: raise e except ValueError as e: @@ -158,6 +161,11 @@ method resolveTxt*( self.nameServers.add(self.nameServers[0]) self.nameServers.delete(0) continue + except Exception as e: + # toString can has a raises: [Exception, ..] because of + # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 + # it can't actually raise though + raiseAssert e.msg debug "Failed to resolve TXT, returning empty set" return @[] From 4f18dd30e9b9aa9c043cf371765df6a37b8a9265 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Thu, 29 Sep 2022 20:02:10 +0200 Subject: [PATCH 07/23] Handle trying to write empty byte seq (#780) --- libp2p/protocols/connectivity/relay/utils.nim | 14 ++++++++------ libp2p/stream/chronosstream.nim | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libp2p/protocols/connectivity/relay/utils.nim b/libp2p/protocols/connectivity/relay/utils.nim index c5449aaa3..275a26f09 100644 --- a/libp2p/protocols/connectivity/relay/utils.nim +++ b/libp2p/protocols/connectivity/relay/utils.nim @@ -64,15 +64,17 @@ proc bridge*(connSrc: Connection, connDst: Connection) {.async.} = await futSrc or futDst if futSrc.finished(): bufRead = await futSrc - bytesSendFromSrcToDst.inc(bufRead) - await connDst.write(@bufSrcToDst[0.. 0: + bytesSendFromSrcToDst.inc(bufRead) + await connDst.write(@bufSrcToDst[0.. 0: + bytesSendFromDstToSrc += bufRead + await connSrc.write(bufDstToSrc[0.. Date: Fri, 30 Sep 2022 10:41:04 +0200 Subject: [PATCH 08/23] RendezVous Protocol (#751) --- libp2p/builders.nim | 11 +- libp2p/protocols/rendezvous.nim | 679 ++++++++++++++++++++++++++++++++ libp2p/utils/offsettedseq.nim | 73 ++++ tests/testnative.nim | 1 + tests/testrendezvous.nim | 125 ++++++ 5 files changed, 888 insertions(+), 1 deletion(-) create mode 100644 libp2p/protocols/rendezvous.nim create mode 100644 libp2p/utils/offsettedseq.nim create mode 100644 tests/testrendezvous.nim diff --git a/libp2p/builders.nim b/libp2p/builders.nim index 5d111b2ee..fdff75ba0 100644 --- a/libp2p/builders.nim +++ b/libp2p/builders.nim @@ -26,7 +26,7 @@ import switch, peerid, peerinfo, stream/connection, multiaddress, crypto/crypto, transports/[transport, tcptransport], muxers/[muxer, mplex/mplex, yamux/yamux], - protocols/[identify, secure/secure, secure/noise], + protocols/[identify, secure/secure, secure/noise, rendezvous], protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport], connmanager, upgrademngrs/muxedupgrade, nameresolving/nameresolver, @@ -60,6 +60,7 @@ type peerStoreCapacity: Option[int] autonat: bool circuitRelay: Relay + rdv: RendezVous proc new*(T: type[SwitchBuilder]): T {.public.} = ## Creates a SwitchBuilder @@ -194,6 +195,10 @@ proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder b.circuitRelay = r b +proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder = + b.rdv = rdv + b + proc build*(b: SwitchBuilder): Switch {.raises: [Defect, LPError], public.} = @@ -261,6 +266,10 @@ proc build*(b: SwitchBuilder): Switch b.circuitRelay.setup(switch) switch.mount(b.circuitRelay) + if not isNil(b.rdv): + b.rdv.setup(switch) + switch.mount(b.rdv) + return switch proc newStandardSwitch*( diff --git a/libp2p/protocols/rendezvous.nim b/libp2p/protocols/rendezvous.nim new file mode 100644 index 000000000..f7d58c95b --- /dev/null +++ b/libp2p/protocols/rendezvous.nim @@ -0,0 +1,679 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import tables, sequtils, sugar, sets, options +import chronos, + chronicles, + bearssl/rand, + stew/[byteutils, objects] +import ./protocol, + ../switch, + ../routing_record, + ../utils/heartbeat, + ../stream/connection, + ../utils/offsettedseq, + ../utils/semaphore + +export chronicles + +logScope: + topics = "libp2p discovery rendezvous" + +const + RendezVousCodec* = "/rendezvous/1.0.0" + MinimumDuration = 2.hours + MaximumDuration = 72.hours + MinimumTTL = MinimumDuration.seconds.uint64 + MaximumTTL = MaximumDuration.seconds.uint64 + RegistrationLimitPerPeer = 1000 + DiscoverLimit = 1000'u64 + SemaphoreDefaultSize = 5 + +type + MessageType {.pure.} = enum + Register = 0 + RegisterResponse = 1 + Unregister = 2 + Discover = 3 + DiscoverResponse = 4 + + ResponseStatus = enum + Ok = 0 + InvalidNamespace = 100 + InvalidSignedPeerRecord = 101 + InvalidTTL = 102 + InvalidCookie = 103 + NotAuthorized = 200 + InternalError = 300 + Unavailable = 400 + + Cookie = object + offset : uint64 + ns : string + + Register = object + ns : string + signedPeerRecord: seq[byte] + ttl: Option[uint64] # in seconds + + RegisterResponse = object + status: ResponseStatus + text: Option[string] + ttl: Option[uint64] # in seconds + + Unregister = object + ns: string + + Discover = object + ns: string + limit: Option[uint64] + cookie: Option[seq[byte]] + + DiscoverResponse = object + registrations: seq[Register] + cookie: Option[seq[byte]] + status: ResponseStatus + text: Option[string] + + Message = object + msgType: MessageType + register: Option[Register] + registerResponse: Option[RegisterResponse] + unregister: Option[Unregister] + discover: Option[Discover] + discoverResponse: Option[DiscoverResponse] + +proc encode(c: Cookie): ProtoBuffer = + result = initProtoBuffer() + result.write(1, c.offset) + result.write(2, c.ns) + result.finish() + +proc encode(r: Register): ProtoBuffer = + result = initProtoBuffer() + result.write(1, r.ns) + result.write(2, r.signedPeerRecord) + if r.ttl.isSome(): + result.write(3, r.ttl.get()) + result.finish() + +proc encode(rr: RegisterResponse): ProtoBuffer = + result = initProtoBuffer() + result.write(1, rr.status.uint) + if rr.text.isSome(): + result.write(2, rr.text.get()) + if rr.ttl.isSome(): + result.write(3, rr.ttl.get()) + result.finish() + +proc encode(u: Unregister): ProtoBuffer = + result = initProtoBuffer() + result.write(1, u.ns) + result.finish() + +proc encode(d: Discover): ProtoBuffer = + result = initProtoBuffer() + result.write(1, d.ns) + if d.limit.isSome(): + result.write(2, d.limit.get()) + if d.cookie.isSome(): + result.write(3, d.cookie.get()) + result.finish() + +proc encode(d: DiscoverResponse): ProtoBuffer = + result = initProtoBuffer() + for reg in d.registrations: + result.write(1, reg.encode()) + if d.cookie.isSome(): + result.write(2, d.cookie.get()) + result.write(3, d.status.uint) + if d.text.isSome(): + result.write(4, d.text.get()) + result.finish() + +proc encode(msg: Message): ProtoBuffer = + result = initProtoBuffer() + result.write(1, msg.msgType.uint) + if msg.register.isSome(): + result.write(2, msg.register.get().encode()) + if msg.registerResponse.isSome(): + result.write(3, msg.registerResponse.get().encode()) + if msg.unregister.isSome(): + result.write(4, msg.unregister.get().encode()) + if msg.discover.isSome(): + result.write(5, msg.discover.get().encode()) + if msg.discoverResponse.isSome(): + result.write(6, msg.discoverResponse.get().encode()) + result.finish() + +proc decode(_: typedesc[Cookie], buf: seq[byte]): Option[Cookie] = + var c: Cookie + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, c.offset) + r2 = pb.getRequiredField(2, c.ns) + if r1.isErr() or r2.isErr(): return none(Cookie) + some(c) + +proc decode(_: typedesc[Register], buf: seq[byte]): Option[Register] = + var + r: Register + ttl: uint64 + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, r.ns) + r2 = pb.getRequiredField(2, r.signedPeerRecord) + r3 = pb.getField(3, ttl) + if r1.isErr() or r2.isErr() or r3.isErr(): return none(Register) + if r3.get(): r.ttl = some(ttl) + some(r) + +proc decode(_: typedesc[RegisterResponse], buf: seq[byte]): Option[RegisterResponse] = + var + rr: RegisterResponse + statusOrd: uint + text: string + ttl: uint64 + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, statusOrd) + r2 = pb.getField(2, text) + r3 = pb.getField(3, ttl) + if r1.isErr() or r2.isErr() or r3.isErr() or + not checkedEnumAssign(rr.status, statusOrd): return none(RegisterResponse) + if r2.get(): rr.text = some(text) + if r3.get(): rr.ttl = some(ttl) + some(rr) + +proc decode(_: typedesc[Unregister], buf: seq[byte]): Option[Unregister] = + var u: Unregister + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, u.ns) + if r1.isErr(): return none(Unregister) + some(u) + +proc decode(_: typedesc[Discover], buf: seq[byte]): Option[Discover] = + var + d: Discover + limit: uint64 + cookie: seq[byte] + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, d.ns) + r2 = pb.getField(2, limit) + r3 = pb.getField(3, cookie) + if r1.isErr() or r2.isErr() or r3.isErr: return none(Discover) + if r2.get(): d.limit = some(limit) + if r3.get(): d.cookie = some(cookie) + some(d) + +proc decode(_: typedesc[DiscoverResponse], buf: seq[byte]): Option[DiscoverResponse] = + var + dr: DiscoverResponse + registrations: seq[seq[byte]] + cookie: seq[byte] + statusOrd: uint + text: string + let + pb = initProtoBuffer(buf) + r1 = pb.getRepeatedField(1, registrations) + r2 = pb.getField(2, cookie) + r3 = pb.getRequiredField(3, statusOrd) + r4 = pb.getField(4, text) + if r1.isErr() or r2.isErr() or r3.isErr or r4.isErr() or + not checkedEnumAssign(dr.status, statusOrd): return none(DiscoverResponse) + for reg in registrations: + var r: Register + let regOpt = Register.decode(reg) + if regOpt.isNone(): return none(DiscoverResponse) + dr.registrations.add(regOpt.get()) + if r2.get(): dr.cookie = some(cookie) + if r4.get(): dr.text = some(text) + some(dr) + +proc decode(_: typedesc[Message], buf: seq[byte]): Option[Message] = + var + msg: Message + statusOrd: uint + pbr, pbrr, pbu, pbd, pbdr: ProtoBuffer + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, statusOrd) + r2 = pb.getField(2, pbr) + r3 = pb.getField(3, pbrr) + r4 = pb.getField(4, pbu) + r5 = pb.getField(5, pbd) + r6 = pb.getField(6, pbdr) + if r1.isErr() or r2.isErr() or r3.isErr() or + r4.isErr() or r5.isErr() or r6.isErr() or + not checkedEnumAssign(msg.msgType, statusOrd): return none(Message) + if r2.get(): + msg.register = Register.decode(pbr.buffer) + if msg.register.isNone(): return none(Message) + if r3.get(): + msg.registerResponse = RegisterResponse.decode(pbrr.buffer) + if msg.registerResponse.isNone(): return none(Message) + if r4.get(): + msg.unregister = Unregister.decode(pbu.buffer) + if msg.unregister.isNone(): return none(Message) + if r5.get(): + msg.discover = Discover.decode(pbd.buffer) + if msg.discover.isNone(): return none(Message) + if r6.get(): + msg.discoverResponse = DiscoverResponse.decode(pbdr.buffer) + if msg.discoverResponse.isNone(): return none(Message) + some(msg) + + +type + RendezVousError* = object of LPError + RegisteredData = object + expiration: Moment + peerId: PeerId + data: Register + + RegisteredSeq = object + s: seq[RegisteredData] + offset: uint64 + + RendezVous* = ref object of LPProtocol + # Registered needs to be an offsetted sequence + # because we need stable index for the cookies. + registered: OffsettedSeq[RegisteredData] + # Namespaces is a table whose key is a salted namespace and + # the value is the index sequence corresponding to this + # namespace in the offsettedqueue. + namespaces: Table[string, seq[int]] + rng: ref HmacDrbgContext + salt: string + defaultDT: Moment + registerDeletionLoop: Future[void] + #registerEvent: AsyncEvent # TODO: to raise during the heartbeat + # + make the heartbeat sleep duration "smarter" + sema: AsyncSemaphore + peers: seq[PeerId] + cookiesSaved: Table[PeerId, Table[string, seq[byte]]] + switch: Switch + +proc checkPeerRecord(spr: seq[byte], peerId: PeerId): Result[void, string] = + if spr.len == 0: return err("Empty peer record") + let signedEnv = ? SignedPeerRecord.decode(spr).mapErr(x => $x) + if signedEnv.data.peerId != peerId: + return err("Bad Peer ID") + return ok() + +proc sendRegisterResponse(conn: Connection, + ttl: uint64) {.async.} = + let msg = encode(Message( + msgType: MessageType.RegisterResponse, + registerResponse: some(RegisterResponse(status: Ok, ttl: some(ttl))))) + await conn.writeLp(msg.buffer) + +proc sendRegisterResponseError(conn: Connection, + status: ResponseStatus, + text: string = "") {.async.} = + let msg = encode(Message( + msgType: MessageType.RegisterResponse, + registerResponse: some(RegisterResponse(status: status, text: some(text))))) + await conn.writeLp(msg.buffer) + +proc sendDiscoverResponse(conn: Connection, + s: seq[Register], + cookie: Cookie) {.async.} = + let msg = encode(Message( + msgType: MessageType.DiscoverResponse, + discoverResponse: some(DiscoverResponse( + status: Ok, + registrations: s, + cookie: some(cookie.encode().buffer) + )) + )) + await conn.writeLp(msg.buffer) + +proc sendDiscoverResponseError(conn: Connection, + status: ResponseStatus, + text: string = "") {.async.} = + let msg = encode(Message( + msgType: MessageType.DiscoverResponse, + discoverResponse: some(DiscoverResponse(status: status, text: some(text))))) + await conn.writeLp(msg.buffer) + +proc countRegister(rdv: RendezVous, peerId: PeerId): int = + let n = Moment.now() + for data in rdv.registered: + if data.peerId == peerId and data.expiration > n: + result.inc() + +proc save(rdv: RendezVous, + ns: string, + peerId: PeerId, + r: Register, + update: bool = true) = + let nsSalted = ns & rdv.salt + discard rdv.namespaces.hasKeyOrPut(nsSalted, newSeq[int]()) + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == peerId: + if update == false: return + rdv.registered[index].expiration = rdv.defaultDT + rdv.registered.add( + RegisteredData( + peerId: peerId, + expiration: Moment.now() + r.ttl.get(MinimumTTL).int64.seconds, + data: r + ) + ) + rdv.namespaces[nsSalted].add(rdv.registered.high) +# rdv.registerEvent.fire() + except KeyError: + doAssert false, "Should have key" + +proc register(rdv: RendezVous, conn: Connection, r: Register): Future[void] = + trace "Received Register", peerId = conn.peerId, ns = r.ns + if r.ns.len notin 1..255: + return conn.sendRegisterResponseError(InvalidNamespace) + let ttl = r.ttl.get(MinimumTTL) + if ttl notin MinimumTTL..MaximumTTL: + return conn.sendRegisterResponseError(InvalidTTL) + let pr = checkPeerRecord(r.signedPeerRecord, conn.peerId) + if pr.isErr(): + return conn.sendRegisterResponseError(InvalidSignedPeerRecord, pr.error()) + if rdv.countRegister(conn.peerId) >= RegistrationLimitPerPeer: + return conn.sendRegisterResponseError(NotAuthorized, "Registration limit reached") + rdv.save(r.ns, conn.peerId, r) + conn.sendRegisterResponse(ttl) + +proc unregister(rdv: RendezVous, conn: Connection, u: Unregister) = + trace "Received Unregister", peerId = conn.peerId, ns = u.ns + let nsSalted = u.ns & rdv.salt + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == conn.peerId: + rdv.registered[index].expiration = rdv.defaultDT + except KeyError: + return + +proc discover(rdv: RendezVous, conn: Connection, d: Discover) {.async.} = + trace "Received Discover", peerId = conn.peerId, ns = d.ns + if d.ns.len notin 0..255: + await conn.sendDiscoverResponseError(InvalidNamespace) + return + var limit = min(DiscoverLimit, d.limit.get(DiscoverLimit)) + var + cookie = + if d.cookie.isSome(): + try: + Cookie.decode(d.cookie.get()).get() + except CatchableError: + await conn.sendDiscoverResponseError(InvalidCookie) + return + else: Cookie(offset: rdv.registered.low().uint64 - 1) + if cookie.ns != d.ns or + cookie.offset notin rdv.registered.low().uint64..rdv.registered.high().uint64: + cookie = Cookie(offset: rdv.registered.low().uint64 - 1) + let + nsSalted = d.ns & rdv.salt + namespaces = + if d.ns != "": + try: + rdv.namespaces[nsSalted] + except KeyError: + await conn.sendDiscoverResponseError(InvalidNamespace) + return + else: toSeq(cookie.offset.int..rdv.registered.high()) + if namespaces.len() == 0: + await conn.sendDiscoverResponse(@[], Cookie()) + return + var offset = namespaces[^1] + let n = Moment.now() + var s = collect(newSeq()): + for index in namespaces: + var reg = rdv.registered[index] + if limit == 0: + offset = index + break + if reg.expiration < n or index.uint64 <= cookie.offset: continue + limit.dec() + reg.data.ttl = some((reg.expiration - Moment.now()).seconds.uint64) + reg.data + rdv.rng.shuffle(s) + await conn.sendDiscoverResponse(s, Cookie(offset: offset.uint64, ns: d.ns)) + +proc advertisePeer(rdv: RendezVous, + peer: PeerId, + msg: seq[byte]) {.async.} = + proc advertiseWrap() {.async.} = + try: + let conn = await rdv.switch.dial(peer, RendezVousCodec) + defer: await conn.close() + await conn.writeLp(msg) + let + buf = await conn.readLp(4096) + msgRecv = Message.decode(buf).get() + if msgRecv.msgType != MessageType.RegisterResponse: + trace "Unexpected register response", peer, msgType = msgRecv.msgType + elif msgRecv.registerResponse.isNone() or + msgRecv.registerResponse.get().status != ResponseStatus.Ok: + trace "Refuse to register", peer, response = msgRecv.registerResponse + except CatchableError as exc: + trace "exception in the advertise", error = exc.msg + finally: + rdv.sema.release() + await rdv.sema.acquire() + discard await advertiseWrap().withTimeout(5.seconds) + +proc advertise*(rdv: RendezVous, + ns: string, + ttl: Duration = MinimumDuration) {.async.} = + let sprBuff = rdv.switch.peerInfo.signedPeerRecord.encode() + if sprBuff.isErr(): + raise newException(RendezVousError, "Wrong Signed Peer Record") + if ns.len notin 1..255: + raise newException(RendezVousError, "Invalid namespace") + if ttl notin MinimumDuration..MaximumDuration: + raise newException(RendezVousError, "Invalid time to live") + let + r = Register(ns: ns, signedPeerRecord: sprBuff.get(), ttl: some(ttl.seconds.uint64)) + msg = encode(Message(msgType: MessageType.Register, register: some(r))) + rdv.save(ns, rdv.switch.peerInfo.peerId, r) + let fut = collect(newSeq()): + for peer in rdv.peers: + trace "Send Advertise", peerId = peer, ns + rdv.advertisePeer(peer, msg.buffer) + await allFutures(fut) + +proc requestLocally*(rdv: RendezVous, ns: string): seq[PeerRecord] = + let + nsSalted = ns & rdv.salt + n = Moment.now() + try: + collect(newSeq()): + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].expiration > n: + SignedPeerRecord.decode(rdv.registered[index].data.signedPeerRecord).get().data + except KeyError as exc: + @[] + +proc request*(rdv: RendezVous, + ns: string, + l: int = DiscoverLimit.int): Future[seq[PeerRecord]] {.async.} = + let nsSalted = ns & rdv.salt + var + s: Table[PeerId, (PeerRecord, Register)] + limit: uint64 + d = Discover(ns: ns) + + if l <= 0 or l > DiscoverLimit.int: + raise newException(RendezVousError, "Invalid limit") + if ns.len notin 0..255: + raise newException(RendezVousError, "Invalid namespace") + limit = l.uint64 + proc requestPeer(peer: PeerId) {.async.} = + let conn = await rdv.switch.dial(peer, RendezVousCodec) + defer: await conn.close() + d.limit = some(limit) + d.cookie = + try: + some(rdv.cookiesSaved[peer][ns]) + except KeyError as exc: + none(seq[byte]) + await conn.writeLp(encode(Message( + msgType: MessageType.Discover, + discover: some(d))).buffer) + let + buf = await conn.readLp(65536) + msgRcv = Message.decode(buf).get() + if msgRcv.msgType != MessageType.DiscoverResponse or + msgRcv.discoverResponse.isNone(): + debug "Unexpected discover response", msgType = msgRcv.msgType + return + let resp = msgRcv.discoverResponse.get() + if resp.status != ResponseStatus.Ok: + trace "Cannot discover", ns, status = resp.status, text = resp.text + return + if resp.cookie.isSome() and resp.cookie.get().len < 1000: + if rdv.cookiesSaved.hasKeyOrPut(peer, {ns: resp.cookie.get()}.toTable): + rdv.cookiesSaved[peer][ns] = resp.cookie.get() + for r in resp.registrations: + if limit == 0: return + if r.ttl.isNone() or r.ttl.get() > MaximumTTL: continue + let sprRes = SignedPeerRecord.decode(r.signedPeerRecord) + if sprRes.isErr(): continue + let pr = sprRes.get().data + if s.hasKey(pr.peerId): + let (prSaved, rSaved) = s[pr.peerId] + if (prSaved.seqNo == pr.seqNo and rSaved.ttl.get() < r.ttl.get()) or + prSaved.seqNo < pr.seqNo: + s[pr.peerId] = (pr, r) + else: + s[pr.peerId] = (pr, r) + limit.dec() + for (_, r) in s.values(): + rdv.save(ns, peer, r, false) + + for peer in rdv.peers: + if limit == 0: break + if RendezVousCodec notin rdv.switch.peerStore[ProtoBook][peer]: continue + try: + trace "Send Request", peerId = peer, ns + await peer.requestPeer() + except CancelledError as exc: + raise exc + except CatchableError as exc: + trace "exception catch in request", error = exc.msg + return toSeq(s.values()).mapIt(it[0]) + +proc unsubscribeLocally*(rdv: RendezVous, ns: string) = + let nsSalted = ns & rdv.salt + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == rdv.switch.peerInfo.peerId: + rdv.registered[index].expiration = rdv.defaultDT + except KeyError: + return + +proc unsubscribe*(rdv: RendezVous, ns: string) {.async.} = + # TODO: find a way to improve this, maybe something similar to the advertise + if ns.len notin 1..255: + raise newException(RendezVousError, "Invalid namespace") + rdv.unsubscribeLocally(ns) + let msg = encode(Message( + msgType: MessageType.Unregister, + unregister: some(Unregister(ns: ns)))) + + proc unsubscribePeer(rdv: RendezVous, peerId: PeerId) {.async.} = + try: + let conn = await rdv.switch.dial(peerId, RendezVousCodec) + defer: await conn.close() + await conn.writeLp(msg.buffer) + except CatchableError as exc: + trace "exception while unsubscribing", error = exc.msg + + for peer in rdv.peers: + discard await rdv.unsubscribePeer(peer).withTimeout(5.seconds) + +proc setup*(rdv: RendezVous, switch: Switch) = + rdv.switch = switch + proc handlePeer(peerId: PeerId, event: PeerEvent) {.async.} = + if event.kind == PeerEventKind.Joined: + rdv.peers.add(peerId) + elif event.kind == PeerEventKind.Left: + rdv.peers.keepItIf(it != peerId) + rdv.switch.addPeerEventHandler(handlePeer, Joined) + rdv.switch.addPeerEventHandler(handlePeer, Left) + +proc new*(T: typedesc[RendezVous], + rng: ref HmacDrbgContext = newRng()): T = + let rdv = T( + rng: rng, + salt: string.fromBytes(generateBytes(rng[], 8)), + registered: initOffsettedSeq[RegisteredData](1), + defaultDT: Moment.now() - 1.days, + #registerEvent: newAsyncEvent(), + sema: newAsyncSemaphore(SemaphoreDefaultSize) + ) + logScope: topics = "libp2p discovery rendezvous" + proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} = + try: + let + buf = await conn.readLp(4096) + msg = Message.decode(buf).get() + case msg.msgType: + of MessageType.Register: await rdv.register(conn, msg.register.get()) + of MessageType.RegisterResponse: + trace "Got an unexpected Register Response", response = msg.registerResponse + of MessageType.Unregister: rdv.unregister(conn, msg.unregister.get()) + of MessageType.Discover: await rdv.discover(conn, msg.discover.get()) + of MessageType.DiscoverResponse: + trace "Got an unexpected Discover Response", response = msg.discoverResponse + except CancelledError as exc: + raise exc + except CatchableError as exc: + trace "exception in rendezvous handler", error = exc.msg + finally: + await conn.close() + + rdv.handler = handleStream + rdv.codec = RendezVousCodec + return rdv + +proc new*(T: typedesc[RendezVous], + switch: Switch, + rng: ref HmacDrbgContext = newRng()): T = + let rdv = T.new(rng) + rdv.setup(switch) + return rdv + +proc deletesRegister(rdv: RendezVous) {.async.} = + heartbeat "Register timeout", 1.minutes: + let n = Moment.now() + rdv.registered.flushIfIt(it.expiration < n) + for data in rdv.namespaces.mvalues(): + data.keepItIf(it >= rdv.registered.offset) + +method start*(rdv: RendezVous) {.async.} = + if not rdv.registerDeletionLoop.isNil: + warn "Starting rendezvous twice" + return + rdv.registerDeletionLoop = rdv.deletesRegister() + rdv.started = true + +method stop*(rdv: RendezVous) {.async.} = + if rdv.registerDeletionLoop.isNil: + warn "Stopping rendezvous without starting it" + return + rdv.started = false + rdv.registerDeletionLoop.cancel() + rdv.registerDeletionLoop = nil diff --git a/libp2p/utils/offsettedseq.nim b/libp2p/utils/offsettedseq.nim new file mode 100644 index 000000000..445953156 --- /dev/null +++ b/libp2p/utils/offsettedseq.nim @@ -0,0 +1,73 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import sequtils + +type + OffsettedSeq*[T] = object + s*: seq[T] + offset*: int + +proc initOffsettedSeq*[T](offset: int = 0): OffsettedSeq[T] = + OffsettedSeq[T](s: newSeq[T](), offset: offset) + +proc all*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool = + o.s.all(pred) + +proc any*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool = + o.s.any(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: T)) = + o.s.apply(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: T): T) = + o.s.apply(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: var T)) = + o.s.apply(pred) + +func count*[T](o: OffsettedSeq[T], x: T): int = + o.s.count(x) + +proc flushIf*[T](o: OffsettedSeq[T], pred: proc (x: T): bool) = + var i = 0 + for e in o.s: + if not pred(e): break + i.inc() + if i > 0: + o.s.delete(0.. 0: + when (NimMajor, NimMinor) < (1, 4): + o.s.delete(0, i - 1) + else: + o.s.delete(0.. Date: Tue, 4 Oct 2022 00:00:00 +0200 Subject: [PATCH 09/23] Discovery interface (#783) Co-authored-by: Tanguy --- libp2p/discovery/discoverymngr.nim | 163 +++++++++++++++++++++++ libp2p/discovery/rendezvousinterface.nim | 77 +++++++++++ libp2p/protocols/rendezvous.nim | 6 +- tests/testdiscovery.nim | 51 +++++++ tests/testnative.nim | 1 + 5 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 libp2p/discovery/discoverymngr.nim create mode 100644 libp2p/discovery/rendezvousinterface.nim create mode 100644 tests/testdiscovery.nim diff --git a/libp2p/discovery/discoverymngr.nim b/libp2p/discovery/discoverymngr.nim new file mode 100644 index 000000000..6c1854361 --- /dev/null +++ b/libp2p/discovery/discoverymngr.nim @@ -0,0 +1,163 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/sequtils +import chronos, chronicles, stew/results +import ../errors + +type + BaseAttr = ref object of RootObj + comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [Defect].} + + Attribute[T] = ref object of BaseAttr + value: T + + PeerAttributes* = object + attributes: seq[BaseAttr] + + DiscoveryService* = distinct string + +proc `==`*(a, b: DiscoveryService): bool {.borrow.} + +proc ofType*[T](f: BaseAttr, _: type[T]): bool = + return f of Attribute[T] + +proc to*[T](f: BaseAttr, _: type[T]): T = + Attribute[T](f).value + +proc add*[T](pa: var PeerAttributes, + value: T) = + pa.attributes.add(Attribute[T]( + value: value, + comparator: proc(f: BaseAttr, c: BaseAttr): bool = + f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T) + ) + ) + +iterator items*(pa: PeerAttributes): BaseAttr = + for f in pa.attributes: + yield f + +proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] = + for f in pa.attributes: + if f.ofType(T): + result.add(f.to(T)) + +proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] = + for f in pa.attributes: + if f.ofType(T): + return Opt.some(f.to(T)) + Opt.none(T) + +proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [Defect, KeyError].} = + pa{T}.valueOr: raise newException(KeyError, "Attritute not found") + +proc match*(pa, candidate: PeerAttributes): bool = + for f in pa.attributes: + block oneAttribute: + for field in candidate.attributes: + if field.comparator(field, f): + break oneAttribute + return false + return true + +type + PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [Defect], gcsafe.} + + DiscoveryInterface* = ref object of RootObj + onPeerFound*: PeerFoundCallback + toAdvertise*: PeerAttributes + advertisementUpdated*: AsyncEvent + advertiseLoop*: Future[void] + +method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} = + doAssert(false, "Not implemented!") + +method advertise*(self: DiscoveryInterface) {.async, base.} = + doAssert(false, "Not implemented!") + +type + DiscoveryError* = object of LPError + + DiscoveryQuery* = ref object + attr: PeerAttributes + peers: AsyncQueue[PeerAttributes] + futs: seq[Future[void]] + + DiscoveryManager* = ref object + interfaces: seq[DiscoveryInterface] + queries: seq[DiscoveryQuery] + +proc add*(dm: DiscoveryManager, di: DiscoveryInterface) = + dm.interfaces &= di + + di.onPeerFound = proc (pa: PeerAttributes) = + for query in dm.queries: + if query.attr.match(pa): + try: + query.peers.putNoWait(pa) + except AsyncQueueFullError as exc: + debug "Cannot push discovered peer to queue" + +proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery = + var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]()) + for i in dm.interfaces: + query.futs.add(i.request(pa)) + dm.queries.add(query) + dm.queries.keepItIf(it.futs.anyIt(not it.finished())) + return query + +proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery = + var pa: PeerAttributes + pa.add(value) + return dm.request(pa) + +proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) = + for i in dm.interfaces: + i.toAdvertise = pa + if i.advertiseLoop.isNil: + i.advertisementUpdated = newAsyncEvent() + i.advertiseLoop = i.advertise() + else: + i.advertisementUpdated.fire() + +proc advertise*[T](dm: DiscoveryManager, value: T) = + var pa: PeerAttributes + pa.add(value) + dm.advertise(pa) + +proc stop*(query: DiscoveryQuery) = + for r in query.futs: + if not r.finished(): r.cancel() + +proc stop*(dm: DiscoveryManager) = + for q in dm.queries: + q.stop() + for i in dm.interfaces: + if isNil(i.advertiseLoop): continue + i.advertiseLoop.cancel() + +proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} = + let getter = query.peers.popFirst() + + try: + await getter or allFinished(query.futs) + except CancelledError as exc: + getter.cancel() + raise exc + + if not finished(getter): + # discovery loops only finish when they don't handle the query + raise newException(DiscoveryError, "Unable to find any peer matching this request") + return await getter diff --git a/libp2p/discovery/rendezvousinterface.nim b/libp2p/discovery/rendezvousinterface.nim new file mode 100644 index 000000000..d3196e87b --- /dev/null +++ b/libp2p/discovery/rendezvousinterface.nim @@ -0,0 +1,77 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import sequtils +import chronos +import ./discoverymngr, + ../protocols/rendezvous, + ../peerid + +type + RendezVousInterface* = ref object of DiscoveryInterface + rdv*: RendezVous + timeToRequest: Duration + timeToAdvertise: Duration + + RdvNamespace* = distinct string + +proc `==`*(a, b: RdvNamespace): bool {.borrow.} + +method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} = + var namespace = "" + for attr in pa: + if attr.ofType(RdvNamespace): + namespace = string attr.to(RdvNamespace) + elif attr.ofType(DiscoveryService): + namespace = string attr.to(DiscoveryService) + elif attr.ofType(PeerId): + namespace = $attr.to(PeerId) + else: + # unhandled type + return + while true: + for pr in await self.rdv.request(namespace): + var peer: PeerAttributes + peer.add(pr.peerId) + for address in pr.addresses: + peer.add(address.address) + + peer.add(DiscoveryService(namespace)) + peer.add(RdvNamespace(namespace)) + self.onPeerFound(peer) + + await sleepAsync(self.timeToRequest) + +method advertise*(self: RendezVousInterface) {.async.} = + while true: + var toAdvertise: seq[string] + for attr in self.toAdvertise: + if attr.ofType(RdvNamespace): + toAdvertise.add string attr.to(RdvNamespace) + elif attr.ofType(DiscoveryService): + toAdvertise.add string attr.to(DiscoveryService) + elif attr.ofType(PeerId): + toAdvertise.add $attr.to(PeerId) + + self.advertisementUpdated.clear() + for toAdv in toAdvertise: + await self.rdv.advertise(toAdv, self.timeToAdvertise) + + await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait() + +proc new*(T: typedesc[RendezVousInterface], + rdv: RendezVous, + ttr: Duration = 1.minutes, + tta: Duration = MinimumDuration): RendezVousInterface = + T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta) diff --git a/libp2p/protocols/rendezvous.nim b/libp2p/protocols/rendezvous.nim index f7d58c95b..5460cdc21 100644 --- a/libp2p/protocols/rendezvous.nim +++ b/libp2p/protocols/rendezvous.nim @@ -32,7 +32,7 @@ logScope: const RendezVousCodec* = "/rendezvous/1.0.0" - MinimumDuration = 2.hours + MinimumDuration* = 2.hours MaximumDuration = 72.hours MinimumTTL = MinimumDuration.seconds.uint64 MaximumTTL = MaximumDuration.seconds.uint64 @@ -284,10 +284,6 @@ type peerId: PeerId data: Register - RegisteredSeq = object - s: seq[RegisteredData] - offset: uint64 - RendezVous* = ref object of LPProtocol # Registered needs to be an offsetted sequence # because we need stable index for the cookies. diff --git a/tests/testdiscovery.nim b/tests/testdiscovery.nim new file mode 100644 index 000000000..58ebf9310 --- /dev/null +++ b/tests/testdiscovery.nim @@ -0,0 +1,51 @@ +{.used.} + +import options, chronos, sets +import stew/byteutils +import ../libp2p/[protocols/rendezvous, + switch, + builders, + discovery/discoverymngr, + discovery/rendezvousinterface,] +import ./helpers + +proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch = + SwitchBuilder.new() + .withRng(newRng()) + .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ]) + .withTcpTransport() + .withMplex() + .withNoise() + .withRendezVous(rdv) + .build() + +suite "Discovery": + teardown: + checkTrackers() + asyncTest "RendezVous test": + let + rdvA = RendezVous.new() + rdvB = RendezVous.new() + clientA = createSwitch(rdvA) + clientB = createSwitch(rdvB) + remoteNode = createSwitch() + dmA = DiscoveryManager() + dmB = DiscoveryManager() + dmA.add(RendezVousInterface.new(rdvA, ttr = 500.milliseconds)) + dmB.add(RendezVousInterface.new(rdvB)) + await allFutures(clientA.start(), clientB.start(), remoteNode.start()) + + await clientB.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs) + await clientA.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs) + + dmB.advertise(RdvNamespace("foo")) + + let + query = dmA.request(RdvNamespace("foo")) + res = await query.getPeer() + check: + res{PeerId}.get() == clientB.peerInfo.peerId + res[PeerId] == clientB.peerInfo.peerId + res.getAll(PeerId) == @[clientB.peerInfo.peerId] + toHashSet(res.getAll(MultiAddress)) == toHashSet(clientB.peerInfo.addrs) + await allFutures(clientA.stop(), clientB.stop(), remoteNode.stop()) diff --git a/tests/testnative.nim b/tests/testnative.nim index 3493c0f9f..f35971b4b 100644 --- a/tests/testnative.nim +++ b/tests/testnative.nim @@ -38,5 +38,6 @@ import testtcptransport, testrelayv1, testrelayv2, testrendezvous, + testdiscovery, testyamux, testautonat From 7b103e02f2fe93ad9e367a9620e5117a18437eb2 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 20 Oct 2022 12:22:28 +0200 Subject: [PATCH 10/23] Allow public address mapping (#767) --- examples/circuitrelay.nim | 14 +++++------ libp2p/peerinfo.nim | 35 ++++++++++++++++++++-------- libp2p/switch.nim | 8 +++---- tests/testidentify.nim | 14 +++++------ tests/testnoise.nim | 3 +-- tests/testpeerinfo.nim | 24 +++++++++++++++---- tests/testrelayv2.nim | 49 ++++++++++++++++++++++++++------------- 7 files changed, 97 insertions(+), 50 deletions(-) diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index bf90d97ba..accbd116b 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -48,19 +48,19 @@ proc main() {.async.} = swSrc = createCircuitRelaySwitch(clSrc) swDst = createCircuitRelaySwitch(clDst) - # Create a relay address to swDst using swRel as the relay - addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" & - $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $swDst.peerInfo.peerId).get() - swDst.mount(proto) await swRel.start() await swSrc.start() await swDst.start() - # Connect both Src and Dst to the relay, but not to each other. - await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs) + let + # Create a relay address to swDst using swRel as the relay + addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" & + $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $swDst.peerInfo.peerId).get() + + # Connect Dst to the relay await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs) # Dst reserve a slot on the relay. diff --git a/libp2p/peerinfo.nim b/libp2p/peerinfo.nim index 59fbe74e7..39a6ad63c 100644 --- a/libp2p/peerinfo.nim +++ b/libp2p/peerinfo.nim @@ -22,11 +22,17 @@ export peerid, multiaddress, crypto, routing_record, errors, results ## Our local peer info type - PeerInfoError* = LPError + PeerInfoError* = object of LPError + + AddressMapper* = + proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] + {.gcsafe, raises: [Defect].} PeerInfo* {.public.} = ref object peerId*: PeerId - addrs*: seq[MultiAddress] + listenAddrs*: seq[MultiAddress] + addrs: seq[MultiAddress] + addressMappers*: seq[AddressMapper] protocols*: seq[string] protoVersion*: string agentVersion*: string @@ -37,6 +43,7 @@ type func shortLog*(p: PeerInfo): auto = ( peerId: $p.peerId, + listenAddrs: mapIt(p.listenAddrs, $it), addrs: mapIt(p.addrs, $it), protocols: mapIt(p.protocols, $it), protoVersion: p.protoVersion, @@ -44,7 +51,11 @@ func shortLog*(p: PeerInfo): auto = ) chronicles.formatIt(PeerInfo): shortLog(it) -proc update*(p: PeerInfo) = +proc update*(p: PeerInfo) {.async.} = + p.addrs = p.listenAddrs + for mapper in p.addressMappers: + p.addrs = await mapper(p.addrs) + let sprRes = SignedPeerRecord.init( p.privateKey, PeerRecord.init(p.peerId, p.addrs) @@ -55,20 +66,25 @@ proc update*(p: PeerInfo) = discard #info "Can't update the signed peer record" +proc addrs*(p: PeerInfo): seq[MultiAddress] = + p.addrs + proc new*( p: typedesc[PeerInfo], key: PrivateKey, - addrs: openArray[MultiAddress] = [], + listenAddrs: openArray[MultiAddress] = [], protocols: openArray[string] = [], protoVersion: string = "", - agentVersion: string = ""): PeerInfo - {.raises: [Defect, PeerInfoError].} = + agentVersion: string = "", + addressMappers = newSeq[AddressMapper](), + ): PeerInfo + {.raises: [Defect, LPError].} = let pubkey = try: key.getPublicKey().tryGet() except CatchableError: raise newException(PeerInfoError, "invalid private key") - + let peerId = PeerId.init(key).tryGet() let peerInfo = PeerInfo( @@ -77,10 +93,9 @@ proc new*( privateKey: key, protoVersion: protoVersion, agentVersion: agentVersion, - addrs: @addrs, + listenAddrs: @listenAddrs, protocols: @protocols, + addressMappers: addressMappers ) - peerInfo.update() - return peerInfo diff --git a/libp2p/switch.nim b/libp2p/switch.nim index 93f2b5f27..b6fc22fec 100644 --- a/libp2p/switch.nim +++ b/libp2p/switch.nim @@ -299,11 +299,11 @@ proc start*(s: Switch) {.async, gcsafe, public.} = trace "starting switch for peer", peerInfo = s.peerInfo var startFuts: seq[Future[void]] for t in s.transports: - let addrs = s.peerInfo.addrs.filterIt( + let addrs = s.peerInfo.listenAddrs.filterIt( t.handles(it) ) - s.peerInfo.addrs.keepItIf( + s.peerInfo.listenAddrs.keepItIf( it notin addrs ) @@ -320,9 +320,9 @@ proc start*(s: Switch) {.async, gcsafe, public.} = for t in s.transports: # for each transport if t.addrs.len > 0 or t.running: s.acceptFuts.add(s.accept(t)) - s.peerInfo.addrs &= t.addrs + s.peerInfo.listenAddrs &= t.addrs - s.peerInfo.update() + await s.peerInfo.update() await s.ms.start() diff --git a/tests/testidentify.nim b/tests/testidentify.nim index 8fc529204..b83eecd41 100644 --- a/tests/testidentify.nim +++ b/tests/testidentify.nim @@ -52,6 +52,9 @@ suite "Identify": msListen = MultistreamSelect.new() msDial = MultistreamSelect.new() + serverFut = transport1.start(ma) + await remotePeerInfo.update() + asyncTeardown: await conn.close() await acceptFut @@ -61,7 +64,6 @@ suite "Identify": asyncTest "default agent version": msListen.addHandler(IdentifyCodec, identifyProto1) - serverFut = transport1.start(ma) proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -84,8 +86,6 @@ suite "Identify": remotePeerInfo.agentVersion = customAgentVersion msListen.addHandler(IdentifyCodec, identifyProto1) - serverFut = transport1.start(ma) - proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -105,7 +105,6 @@ suite "Identify": asyncTest "handle failed identify": msListen.addHandler(IdentifyCodec, identifyProto1) - asyncSpawn transport1.start(ma) proc acceptHandler() {.async.} = var conn: Connection @@ -128,7 +127,6 @@ suite "Identify": asyncTest "can send signed peer record": msListen.addHandler(IdentifyCodec, identifyProto1) identifyProto1.sendSignedPeerRecord = true - serverFut = transport1.start(ma) proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -195,7 +193,8 @@ suite "Identify": asyncTest "simple push identify": switch2.peerInfo.protocols.add("/newprotocol/") - switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + await switch2.peerInfo.update() check: switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs @@ -216,7 +215,8 @@ suite "Identify": asyncTest "wrong peer id push identify": switch2.peerInfo.protocols.add("/newprotocol/") - switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + await switch2.peerInfo.update() check: switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs diff --git a/tests/testnoise.nim b/tests/testnoise.nim index a94714ce0..f715a7b33 100644 --- a/tests/testnoise.nim +++ b/tests/testnoise.nim @@ -60,8 +60,7 @@ method init(p: TestProto) {.gcsafe.} = proc createSwitch(ma: MultiAddress; outgoing: bool, secio: bool = false): (Switch, PeerInfo) = var privateKey = PrivateKey.random(ECDSA, rng[]).get() - peerInfo = PeerInfo.new(privateKey) - peerInfo.addrs.add(ma) + peerInfo = PeerInfo.new(privateKey, @[ma]) proc createMplex(conn: Connection): Muxer = result = Mplex.new(conn) diff --git a/tests/testpeerinfo.nim b/tests/testpeerinfo.nim index 339836b0d..e4f273997 100644 --- a/tests/testpeerinfo.nim +++ b/tests/testpeerinfo.nim @@ -18,22 +18,24 @@ suite "PeerInfo": check peerId == peerInfo.peerId check seckey.getPublicKey().get() == peerInfo.publicKey - + test "Signed peer record": const ExpectedDomain = $multiCodec("libp2p-peer-record") ExpectedPayloadType = @[(byte) 0x03, (byte) 0x01] - + let seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey).get() multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()] peerInfo = PeerInfo.new(seckey, multiAddresses) - + + waitFor(peerInfo.update()) + let env = peerInfo.signedPeerRecord.envelope rec = PeerRecord.decode(env.payload()).tryGet() - + # Check envelope fields check: env.publicKey == peerInfo.publicKey @@ -47,3 +49,17 @@ suite "PeerInfo": rec.addresses.len == 2 rec.addresses[0].address == multiAddresses[0] rec.addresses[1].address == multiAddresses[1] + + test "Public address mapping": + let + seckey = PrivateKey.random(ECDSA, rng[]).get() + multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()] + multiAddresses2 = @[MultiAddress.init("/ip4/8.8.8.8/tcp/33").tryGet()] + + proc addressMapper(input: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} = + check input == multiAddresses + await sleepAsync(0.seconds) + return multiAddresses2 + var peerInfo = PeerInfo.new(seckey, multiAddresses, addressMappers = @[addressMapper]) + waitFor peerInfo.update() + check peerInfo.addrs == multiAddresses2 diff --git a/tests/testrelayv2.nim b/tests/testrelayv2.nim index 8565b36d9..16d37d970 100644 --- a/tests/testrelayv2.nim +++ b/tests/testrelayv2.nim @@ -118,7 +118,6 @@ suite "Circuit Relay V2": asyncTeardown: checkTrackers() var - addrs {.threadvar.}: MultiAddress customProtoCodec {.threadvar.}: string proto {.threadvar.}: LPProtocol ttl {.threadvar.}: int @@ -145,9 +144,6 @@ suite "Circuit Relay V2": src = createSwitch(srcCl) dst = createSwitch(dstCl) rel = newStandardSwitch() - addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & - $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $dst.peerInfo.peerId).get() asyncTest "Connection succeed": proto.handler = proc(conn: Connection, proto: string) {.async.} = @@ -167,6 +163,10 @@ suite "Circuit Relay V2": await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -200,6 +200,10 @@ suite "Circuit Relay V2": await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -245,6 +249,10 @@ take to the ship.""") await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -277,6 +285,10 @@ take to the ship.""") await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -308,11 +320,6 @@ take to the ship.""") rel2Cl = RelayClient.new(canHop = true) rel2 = createSwitch(rel2Cl) rv2 = Relay.new() - addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & - $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $rel2.peerInfo.peerId & "/p2p/" & - $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" & - $dst.peerInfo.peerId).get() ] rv2.setup(rel) rel.mount(rv2) dst.mount(proto) @@ -321,6 +328,13 @@ take to the ship.""") await src.start() await dst.start() + let + addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $rel2.peerInfo.peerId & "/p2p/" & + $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() ] + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await rel2.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel2.peerInfo.peerId, rel2.peerInfo.addrs) @@ -367,6 +381,16 @@ take to the ship.""") switchA = createSwitch(clientA) switchB = createSwitch(clientB) switchC = createSwitch(clientC) + + switchA.mount(protoBCA) + switchB.mount(protoCAB) + switchC.mount(protoABC) + + await switchA.start() + await switchB.start() + await switchC.start() + + let addrsABC = MultiAddress.init($switchB.peerInfo.addrs[0] & "/p2p/" & $switchB.peerInfo.peerId & "/p2p-circuit/p2p/" & $switchC.peerInfo.peerId).get() @@ -376,13 +400,6 @@ take to the ship.""") addrsCAB = MultiAddress.init($switchA.peerInfo.addrs[0] & "/p2p/" & $switchA.peerInfo.peerId & "/p2p-circuit/p2p/" & $switchB.peerInfo.peerId).get() - switchA.mount(protoBCA) - switchB.mount(protoCAB) - switchC.mount(protoABC) - - await switchA.start() - await switchB.start() - await switchC.start() await switchA.connect(switchB.peerInfo.peerId, switchB.peerInfo.addrs) await switchB.connect(switchC.peerInfo.peerId, switchC.peerInfo.addrs) From a086fcba7201f486cab90ab69d637698d8b82482 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 20 Oct 2022 14:52:02 +0200 Subject: [PATCH 11/23] Remove shallow copies (#782) --- libp2p/crypto/minasn1.nim | 32 ++++++++++++-------------------- libp2p/multiaddress.nim | 16 ++++------------ libp2p/peerid.nim | 2 +- libp2p/protobuf/minprotobuf.nim | 2 +- libp2p/vbuffer.nim | 14 +------------- 5 files changed, 19 insertions(+), 47 deletions(-) diff --git a/libp2p/crypto/minasn1.nim b/libp2p/crypto/minasn1.nim index cb30b662d..4c7966d3b 100644 --- a/libp2p/crypto/minasn1.nim +++ b/libp2p/crypto/minasn1.nim @@ -528,8 +528,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass, index: ttag, offset: int(ab.offset), - length: 1) - shallowCopy(field.buffer, ab.buffer) + length: 1, buffer: ab.buffer) field.vbool = (b == 0xFF'u8) ab.offset += 1 return ok(field) @@ -554,8 +553,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Negative or Positive integer field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8: # Negative integer if length <= 8: @@ -579,16 +577,15 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Zero value integer field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length), vint: 0'u64) - shallowCopy(field.buffer, ab.buffer) + length: int(length), vint: 0'u64, + buffer: ab.buffer) ab.offset += int(length) return ok(field) else: # Positive integer with leading zero field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset) + 1, - length: int(length) - 1) - shallowCopy(field.buffer, ab.buffer) + length: int(length) - 1, buffer: ab.buffer) if length <= 9: for i in 1 ..< int(length): field.vint = (field.vint shl 8) or @@ -610,8 +607,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Zero-length BIT STRING. field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass, index: ttag, offset: int(ab.offset + 1), - length: 0, ubits: 0) - shallowCopy(field.buffer, ab.buffer) + length: 0, ubits: 0, buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -631,8 +627,8 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass, index: ttag, offset: int(ab.offset + 1), - length: int(length - 1), ubits: int(unused)) - shallowCopy(field.buffer, ab.buffer) + length: int(length - 1), ubits: int(unused), + buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -643,8 +639,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -654,8 +649,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = return err(Asn1Error.Incorrect) field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag, - offset: int(ab.offset), length: 0) - shallowCopy(field.buffer, ab.buffer) + offset: int(ab.offset), length: 0, buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -666,8 +660,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -678,8 +671,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index fd663f94a..d7ef6e9cc 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -516,15 +516,10 @@ proc trimRight(s: string, ch: char): string = break result = s[0..(s.high - m)] -proc shcopy*(m1: var MultiAddress, m2: MultiAddress) = - shallowCopy(m1.data.buffer, m2.data.buffer) - m1.data.offset = m2.data.offset - proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] = ## Returns MultiAddress ``ma`` protocol code. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -537,8 +532,7 @@ proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] = proc protoName*(ma: MultiAddress): MaResult[string] = ## Returns MultiAddress ``ma`` protocol name. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -555,9 +549,8 @@ proc protoArgument*(ma: MultiAddress, ## If current MultiAddress do not have argument value, then result will be ## ``0``. var header: uint64 - var vb: MultiAddress + var vb = ma var buffer: seq[byte] - shcopy(vb, ma) if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -792,8 +785,7 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string, proc validate*(ma: MultiAddress): bool = ## Returns ``true`` if MultiAddress ``ma`` is valid. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma while true: if vb.data.isEmpty(): break diff --git a/libp2p/peerid.nim b/libp2p/peerid.nim index 401fee96d..e9fac8e4a 100644 --- a/libp2p/peerid.nim +++ b/libp2p/peerid.nim @@ -148,7 +148,7 @@ func init*(pid: var PeerId, data: string): bool = if Base58.decode(data, p, length) == Base58Status.Success: p.setLen(length) var opid: PeerId - shallowCopy(opid.data, p) + opid.data = p if opid.validate(): pid = opid result = true diff --git a/libp2p/protobuf/minprotobuf.nim b/libp2p/protobuf/minprotobuf.nim index e1e36bd61..57223b0e8 100644 --- a/libp2p/protobuf/minprotobuf.nim +++ b/libp2p/protobuf/minprotobuf.nim @@ -124,7 +124,7 @@ proc vsizeof*(field: ProtoField): int {.inline.} = proc initProtoBuffer*(data: seq[byte], offset = 0, options: set[ProtoFlags] = {}): ProtoBuffer = ## Initialize ProtoBuffer with shallow copy of ``data``. - shallowCopy(result.buffer, data) + result.buffer = data result.offset = offset result.options = options diff --git a/libp2p/vbuffer.nim b/libp2p/vbuffer.nim index 7a834ad23..0b5b9ceec 100644 --- a/libp2p/vbuffer.nim +++ b/libp2p/vbuffer.nim @@ -39,21 +39,9 @@ proc len*(vb: VBuffer): int = result = len(vb.buffer) - vb.offset doAssert(result >= 0) -proc isLiteral[T](s: seq[T]): bool {.inline.} = - when defined(gcOrc) or defined(gcArc): - false - else: - type - SeqHeader = object - length, reserved: int - (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 - proc initVBuffer*(data: seq[byte], offset = 0): VBuffer = ## Initialize VBuffer with shallow copy of ``data``. - if isLiteral(data): - result.buffer = data - else: - shallowCopy(result.buffer, data) + result.buffer = data result.offset = offset proc initVBuffer*(data: openArray[byte], offset = 0): VBuffer = From 2e12c7ab730eaef6f5e870d1542d64ff7760b005 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 21 Oct 2022 16:59:53 +0200 Subject: [PATCH 12/23] Temporarily remove failing test (#788) --- tests/pubsub/testpubsub.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pubsub/testpubsub.nim b/tests/pubsub/testpubsub.nim index 08b9abaae..0e910cbcf 100644 --- a/tests/pubsub/testpubsub.nim +++ b/tests/pubsub/testpubsub.nim @@ -2,9 +2,11 @@ import ../stublogger -import testfloodsub, - testgossipsub, - testgossipsub2, +import testfloodsub +when not defined(linux): + import testgossipsub, + testgossipsub2 +import testmcache, testtimedcache, testmessage From 4b105c6abde61cf6dbb96f2cb474c13a3a77ed2e Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 21 Oct 2022 17:00:36 +0200 Subject: [PATCH 13/23] GossipSub tutorial (#784) --- examples/tutorial_4_gossipsub.nim | 163 ++++++++++++++++++++ libp2p.nimble | 4 +- libp2p/protocols/pubsub/gossipsub/types.nim | 6 +- mkdocs.yml | 1 + 4 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 examples/tutorial_4_gossipsub.nim diff --git a/examples/tutorial_4_gossipsub.nim b/examples/tutorial_4_gossipsub.nim new file mode 100644 index 000000000..0e8274caf --- /dev/null +++ b/examples/tutorial_4_gossipsub.nim @@ -0,0 +1,163 @@ +## # GossipSub +## +## In this tutorial, we'll build a simple GossipSub network +## to broadcast the metrics we built in the previous tutorial. +## +## GossipSub is used to broadcast some messages in a network, +## and allows to balance between latency, bandwidth usage, +## privacy and attack resistance. +## +## You'll find a good explanation on how GossipSub works +## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot +## of parameters you can tweak to adjust how GossipSub behaves but here we'll +## use the sane defaults shipped with libp2p. +## +## We'll start by creating our metric structure like previously + +import chronos +import stew/results + +import libp2p +import libp2p/protocols/pubsub/rpc/messages + +type + Metric = object + name: string + value: float + + MetricList = object + hostname: string + metrics: seq[Metric] + +{.push raises: [].} + +proc encode(m: Metric): ProtoBuffer = + result = initProtoBuffer() + result.write(1, m.name) + result.write(2, m.value) + result.finish() + +proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] = + var res: Metric + let pb = initProtoBuffer(buf) + discard ? pb.getField(1, res.name) + discard ? pb.getField(2, res.value) + ok(res) + +proc encode(m: MetricList): ProtoBuffer = + result = initProtoBuffer() + for metric in m.metrics: + result.write(1, metric.encode()) + result.write(2, m.hostname) + result.finish() + +proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] = + var + res: MetricList + metrics: seq[seq[byte]] + let pb = initProtoBuffer(buf) + discard ? pb.getRepeatedField(1, metrics) + + for metric in metrics: + res.metrics &= ? Metric.decode(metric) + ? pb.getRequiredField(2, res.hostname) + ok(res) + +## This is exactly like the previous structure, except that we added +## a `hostname` to distinguish where the metric is coming from. +## +## Now we'll create a small GossipSub network to broadcast the metrics, +## and collect them on one of the node. + +type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string] + +proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} = + # This procedure will handle one of the node of the network + node.gossip.addValidator(["metrics"], + proc(topic: string, message: Message): Future[ValidationResult] {.async.} = + let decoded = MetricList.decode(message.data) + if decoded.isErr: return ValidationResult.Reject + return ValidationResult.Accept + ) + # This "validator" will attach to the `metrics` topic and make sure + # that every message in this topic is valid. This allows us to stop + # propagation of invalid messages quickly in the network, and punish + # peers sending them. + + # `John` will be responsible to log the metrics, the rest of the nodes + # will just forward them in the network + if node.hostname == "John": + node.gossip.subscribe("metrics", + proc (topic: string, data: seq[byte]) {.async.} = + echo MetricList.decode(data).tryGet() + ) + else: + node.gossip.subscribe("metrics", nil) + + # Create random metrics 10 times and broadcast them + for _ in 0..<10: + await sleepAsync(500.milliseconds) + var metricList = MetricList(hostname: node.hostname) + let metricCount = rng[].generate(uint32) mod 4 + for i in 0 ..< metricCount + 1: + metricList.metrics.add(Metric( + name: "metric_" & $i, + value: float(rng[].generate(uint16)) / 1000.0 + )) + + discard await node.gossip.publish("metrics", encode(metricList).buffer) + await node.switch.stop() + +## For our main procedure, we'll create a few nodes, and connect them together. +## Note that they are not all interconnected, but GossipSub will take care of +## broadcasting to the full network nonetheless. +proc main {.async.} = + let rng = newRng() + var nodes: seq[Node] + + for hostname in ["John", "Walter", "David", "Thuy", "Amy"]: + let + switch = newStandardSwitch(rng=rng) + gossip = GossipSub.init(switch = switch, triggerSelf = true) + switch.mount(gossip) + await switch.start() + + nodes.add((switch, gossip, hostname)) + + for index, node in nodes: + # Connect to a few neighbors + for otherNodeIdx in index - 1 .. index + 2: + if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index: continue + let otherNode = nodes[otherNodeIdx] + await node.switch.connect( + otherNode.switch.peerInfo.peerId, + otherNode.switch.peerInfo.addrs) + + var allFuts: seq[Future[void]] + for node in nodes: + allFuts.add(oneNode(node, rng)) + + await allFutures(allFuts) + +waitFor(main()) + +## If you run this program, you should see something like: +## ``` +## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)]) +## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)]) +## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)]) +## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)]) +## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)]) +## ``` +## +## This is John receiving & logging everyone's metrics. +## +## ## Going further +## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams) +## and [topic params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams), +## you can achieve very different properties. +## +## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4) +## +## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub, +## and adds features such as history, spam protection, and light node friendliness. diff --git a/libp2p.nimble b/libp2p.nimble index eb661d84e..568df0046 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -90,6 +90,7 @@ task website, "Build the website": tutorialToMd("examples/tutorial_1_connect.nim") tutorialToMd("examples/tutorial_2_customproto.nim") tutorialToMd("examples/tutorial_3_protobuf.nim") + tutorialToMd("examples/tutorial_4_gossipsub.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -100,8 +101,9 @@ task examples_build, "Build the samples": buildSample("tutorial_1_connect", true) buildSample("tutorial_2_customproto", true) if (NimMajor, NimMinor) > (1, 2): - # This tutorial relies on post 1.4 exception tracking + # These tutorials relies on post 1.4 exception tracking buildSample("tutorial_3_protobuf", true) + buildSample("tutorial_4_gossipsub", true) # pin system # while nimble lockfile diff --git a/libp2p/protocols/pubsub/gossipsub/types.nim b/libp2p/protocols/pubsub/gossipsub/types.nim index 5142aef9c..5bc96ea8b 100644 --- a/libp2p/protocols/pubsub/gossipsub/types.nim +++ b/libp2p/protocols/pubsub/gossipsub/types.nim @@ -16,7 +16,7 @@ import chronos import std/[tables, sets] import ".."/[floodsub, peertable, mcache, pubsubpeer] import "../rpc"/[messages] -import "../../.."/[peerid, multiaddress] +import "../../.."/[peerid, multiaddress, utility] const GossipSubCodec* = "/meshsub/1.1.0" @@ -65,7 +65,7 @@ type meshFailurePenalty*: float64 invalidMessageDeliveries*: float64 - TopicParams* = object + TopicParams* {.public.} = object topicWeight*: float64 # p1 @@ -102,7 +102,7 @@ type appScore*: float64 # application specific score behaviourPenalty*: float64 # the eventual penalty score - GossipSubParams* = object + GossipSubParams* {.public.} = object explicit*: bool pruneBackoff*: Duration unsubscribeBackoff*: Duration diff --git a/mkdocs.yml b/mkdocs.yml index 57d847994..c1906f3a3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,5 +45,6 @@ nav: - 'Simple connection': tutorial_1_connect.md - 'Create a custom protocol': tutorial_2_customproto.md - 'Protobuf': tutorial_3_protobuf.md + - 'GossipSub': tutorial_4_gossipsub.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' From dc13ff81d30a7b6866f58b958bf7477244393560 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 26 Oct 2022 11:07:31 +0200 Subject: [PATCH 14/23] Revert "Temporarily remove failing test (#788)" & fix tests (#789) Co-authored-by: Ludovic Chenut --- libp2p/protocols/pubsub/gossipsub.nim | 2 +- tests/pubsub/testgossipsub.nim | 33 +++++--------------- tests/pubsub/testgossipsub2.nim | 28 +++-------------- tests/pubsub/testpubsub.nim | 8 ++--- tests/pubsub/utils.nim | 43 ++++++++++++++++++++++++++- 5 files changed, 57 insertions(+), 57 deletions(-) diff --git a/libp2p/protocols/pubsub/gossipsub.nim b/libp2p/protocols/pubsub/gossipsub.nim index fb1873c8c..92d352655 100644 --- a/libp2p/protocols/pubsub/gossipsub.nim +++ b/libp2p/protocols/pubsub/gossipsub.nim @@ -360,7 +360,7 @@ method rpcHandler*(g: GossipSub, template sub: untyped = rpcMsg.subscriptions[i] g.handleSubscribe(peer, sub.topic, sub.subscribe) - # the above call applied limtis to subs number + # the above call applied limits to subs number # in gossipsub we want to apply scoring as well if rpcMsg.subscriptions.len > g.topicsHigh: debug "received an rpc message with an oversized amount of subscriptions", peer, diff --git a/tests/pubsub/testgossipsub.nim b/tests/pubsub/testgossipsub.nim index 8c4a7667d..83e3af74f 100644 --- a/tests/pubsub/testgossipsub.nim +++ b/tests/pubsub/testgossipsub.nim @@ -9,7 +9,7 @@ {.used.} -import sequtils, options, tables, sets +import sequtils, options, tables, sets, sugar import chronos, stew/byteutils import chronicles import utils, ../../libp2p/[errors, @@ -29,26 +29,6 @@ import ../helpers proc `$`(peer: PubSubPeer): string = shortLog(peer) -proc waitSub(sender, receiver: auto; key: string) {.async, gcsafe.} = - if sender == receiver: - return - let timeout = Moment.now() + 5.seconds - let fsub = GossipSub(sender) - - # this is for testing purposes only - # peers can be inside `mesh` and `fanout`, not just `gossipsub` - while (not fsub.gossipsub.hasKey(key) or - not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and - (not fsub.mesh.hasKey(key) or - not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and - (not fsub.fanout.hasKey(key) or - not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)): - trace "waitSub sleeping..." - - # await - await sleepAsync(5.milliseconds) - doAssert Moment.now() < timeout, "waitSub timeout!" - template tryPublish(call: untyped, require: int, wait = 10.milliseconds, timeout = 5.seconds): untyped = var expiration = Moment.now() + timeout @@ -690,14 +670,14 @@ suite "GossipSub": seenFut.complete() dialer.subscribe("foobar", handler) - await waitSub(nodes[0], dialer, "foobar") + await waitSubGraph(nodes, "foobar") tryPublish await wait(nodes[0].publish("foobar", toBytes("from node " & $nodes[0].peerInfo.peerId)), 1.minutes), 1 - await wait(seenFut, 2.minutes) + await wait(seenFut, 1.minutes) check: seen.len >= runs for k, v in seen.pairs: check: v >= 1 @@ -726,10 +706,11 @@ suite "GossipSub": var seen: Table[string, int] var seenFut = newFuture[void]() + for i in 0..= runs for k, v in seen.pairs: check: v >= 1 diff --git a/tests/pubsub/testgossipsub2.nim b/tests/pubsub/testgossipsub2.nim index 3f21196e0..87318a4fd 100644 --- a/tests/pubsub/testgossipsub2.nim +++ b/tests/pubsub/testgossipsub2.nim @@ -10,8 +10,7 @@ {.used.} import sequtils, options, tables, sets -import chronos, stew/byteutils -import chronicles +import chronos, stew/byteutils, chronicles import utils, ../../libp2p/[errors, peerid, peerinfo, @@ -25,26 +24,6 @@ import utils, ../../libp2p/[errors, protocols/pubsub/rpc/messages] import ../helpers -proc waitSub(sender, receiver: auto; key: string) {.async, gcsafe.} = - if sender == receiver: - return - let timeout = Moment.now() + 5.seconds - let fsub = GossipSub(sender) - - # this is for testing purposes only - # peers can be inside `mesh` and `fanout`, not just `gossipsub` - while (not fsub.gossipsub.hasKey(key) or - not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and - (not fsub.mesh.hasKey(key) or - not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and - (not fsub.fanout.hasKey(key) or - not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)): - trace "waitSub sleeping..." - - # await - await sleepAsync(5.milliseconds) - doAssert Moment.now() < timeout, "waitSub timeout!" - template tryPublish(call: untyped, require: int, wait = 10.milliseconds, timeout = 10.seconds): untyped = var expiration = Moment.now() + timeout @@ -269,7 +248,7 @@ suite "GossipSub": await allFuturesThrowing(nodesFut.concat()) - asyncTest "GossipsSub peers disconnections mechanics": + asyncTest "GossipSub peers disconnections mechanics": var runs = 10 let @@ -294,7 +273,8 @@ suite "GossipSub": seenFut.complete() dialer.subscribe("foobar", handler) - await waitSub(nodes[0], dialer, "foobar") + + await waitSubGraph(nodes, "foobar") # ensure peer stats are stored properly and kept properly check: diff --git a/tests/pubsub/testpubsub.nim b/tests/pubsub/testpubsub.nim index 0e910cbcf..08b9abaae 100644 --- a/tests/pubsub/testpubsub.nim +++ b/tests/pubsub/testpubsub.nim @@ -2,11 +2,9 @@ import ../stublogger -import testfloodsub -when not defined(linux): - import testgossipsub, - testgossipsub2 -import +import testfloodsub, + testgossipsub, + testgossipsub2, testmcache, testtimedcache, testmessage diff --git a/tests/pubsub/utils.nim b/tests/pubsub/utils.nim index 59918a70e..095c68c45 100644 --- a/tests/pubsub/utils.nim +++ b/tests/pubsub/utils.nim @@ -4,7 +4,7 @@ const libp2p_pubsub_verify {.booldefine.} = true libp2p_pubsub_anonymize {.booldefine.} = false -import hashes, random, tables +import hashes, random, tables, sets, sequtils import chronos, stew/[byteutils, results] import ../../libp2p/[builders, protocols/pubsub/errors, @@ -13,6 +13,7 @@ import ../../libp2p/[builders, protocols/pubsub/floodsub, protocols/pubsub/rpc/messages, protocols/secure/secure] +import chronicles export builders @@ -102,3 +103,43 @@ proc subscribeRandom*(nodes: seq[PubSub]) {.async.} = if dialer.peerInfo.peerId != node.peerInfo.peerId: await dialer.switch.connect(node.peerInfo.peerId, node.peerInfo.addrs) dialed.add(node.peerInfo.peerId) + +proc waitSub*(sender, receiver: auto; key: string) {.async, gcsafe.} = + if sender == receiver: + return + let timeout = Moment.now() + 5.seconds + let fsub = GossipSub(sender) + + # this is for testing purposes only + # peers can be inside `mesh` and `fanout`, not just `gossipsub` + while (not fsub.gossipsub.hasKey(key) or + not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and + (not fsub.mesh.hasKey(key) or + not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and + (not fsub.fanout.hasKey(key) or + not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)): + trace "waitSub sleeping..." + + # await + await sleepAsync(5.milliseconds) + doAssert Moment.now() < timeout, "waitSub timeout!" + +proc waitSubGraph*(nodes: seq[PubSub], key: string) {.async, gcsafe.} = + let timeout = Moment.now() + 5.seconds + while true: + var + nodesMesh: Table[PeerId, seq[PeerId]] + seen: HashSet[PeerId] + for n in nodes: + nodesMesh[n.peerInfo.peerId] = toSeq(GossipSub(n).mesh.getOrDefault(key).items()).mapIt(it.peerId) + proc explore(p: PeerId) = + if p in seen: return + seen.incl(p) + for peer in nodesMesh.getOrDefault(p): + explore(peer) + explore(nodes[0].peerInfo.peerId) + if seen.len == nodes.len: return + trace "waitSubGraph sleeping..." + + await sleepAsync(5.milliseconds) + doAssert Moment.now() < timeout, "waitSubGraph timeout!" From 4bce8f38c9b9c55c8bf0dc88cf5843662d85d78f Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 28 Oct 2022 01:10:24 +0200 Subject: [PATCH 15/23] Remove sleepAsync from tests/testswitch (#792) --- tests/helpers.nim | 21 ++++++------ tests/pubsub/testfloodsub.nim | 2 +- tests/pubsub/testgossipsub.nim | 7 ++-- tests/testidentify.nim | 4 +-- tests/testmplex.nim | 8 ++--- tests/testswitch.nim | 59 ++++++---------------------------- 6 files changed, 29 insertions(+), 72 deletions(-) diff --git a/tests/helpers.nim b/tests/helpers.nim index b82011f7c..d79ca72f9 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -105,16 +105,15 @@ proc bridgedConnections*: (Connection, Connection) = return (connA, connB) -proc checkExpiringInternal(cond: proc(): bool {.raises: [Defect].} ): Future[bool] {.async, gcsafe.} = - {.gcsafe.}: - let start = Moment.now() - while true: - if Moment.now() > (start + chronos.seconds(5)): - return false - elif cond(): - return true - else: - await sleepAsync(1.millis) +proc checkExpiringInternal(cond: proc(): bool {.raises: [Defect], gcsafe.} ): Future[bool] {.async, gcsafe.} = + let start = Moment.now() + while true: + if Moment.now() > (start + chronos.seconds(5)): + return false + elif cond(): + return true + else: + await sleepAsync(1.millis) template checkExpiring*(code: untyped): untyped = - checkExpiringInternal(proc(): bool = code) + check await checkExpiringInternal(proc(): bool = code) diff --git a/tests/pubsub/testfloodsub.nim b/tests/pubsub/testfloodsub.nim index dad332190..0606618a6 100644 --- a/tests/pubsub/testfloodsub.nim +++ b/tests/pubsub/testfloodsub.nim @@ -351,7 +351,7 @@ suite "FloodSub": check (await smallNode[0].publish("foo", smallMessage1)) > 0 check (await bigNode[0].publish("foo", smallMessage2)) > 0 - check (await checkExpiring(messageReceived == 2)) == true + checkExpiring: messageReceived == 2 check (await smallNode[0].publish("foo", bigMessage)) > 0 check (await bigNode[0].publish("foo", bigMessage)) > 0 diff --git a/tests/pubsub/testgossipsub.nim b/tests/pubsub/testgossipsub.nim index 83e3af74f..ab6df3af1 100644 --- a/tests/pubsub/testgossipsub.nim +++ b/tests/pubsub/testgossipsub.nim @@ -316,11 +316,10 @@ suite "GossipSub": let gossip1 = GossipSub(nodes[0]) let gossip2 = GossipSub(nodes[1]) - check await checkExpiring( + checkExpiring: "foobar" in gossip2.topics and "foobar" in gossip1.gossipsub and gossip1.gossipsub.hasPeerId("foobar", gossip2.peerInfo.peerId) - ) await allFuturesThrowing( nodes[0].switch.stop(), @@ -463,7 +462,7 @@ suite "GossipSub": nodes[0].unsubscribe("foobar", handler) let gsNode = GossipSub(nodes[1]) - check await checkExpiring(gsNode.mesh.getOrDefault("foobar").len == 0) + checkExpiring: gsNode.mesh.getOrDefault("foobar").len == 0 nodes[0].subscribe("foobar", handler) @@ -582,7 +581,7 @@ suite "GossipSub": gossip1.seen = TimedCache[MessageId].init() gossip3.seen = TimedCache[MessageId].init() let msgId = toSeq(gossip2.validationSeen.keys)[0] - check await checkExpiring(try: gossip2.validationSeen[msgId].len > 0 except: false) + checkExpiring(try: gossip2.validationSeen[msgId].len > 0 except: false) result = ValidationResult.Accept bFinished.complete() diff --git a/tests/testidentify.nim b/tests/testidentify.nim index b83eecd41..b7bd127e0 100644 --- a/tests/testidentify.nim +++ b/tests/testidentify.nim @@ -202,8 +202,8 @@ suite "Identify": await identifyPush2.push(switch2.peerInfo, conn) - check await checkExpiring(switch1.peerStore[ProtoBook][switch2.peerInfo.peerId] == switch2.peerInfo.protocols) - check await checkExpiring(switch1.peerStore[AddressBook][switch2.peerInfo.peerId] == switch2.peerInfo.addrs) + checkExpiring: switch1.peerStore[ProtoBook][switch2.peerInfo.peerId] == switch2.peerInfo.protocols + checkExpiring: switch1.peerStore[AddressBook][switch2.peerInfo.peerId] == switch2.peerInfo.addrs await closeAll() diff --git a/tests/testmplex.nim b/tests/testmplex.nim index 0f68f7d61..85698630e 100644 --- a/tests/testmplex.nim +++ b/tests/testmplex.nim @@ -816,7 +816,7 @@ suite "Mplex": for i in 0..9: dialStreams.add((await mplexDial.newStream())) - check await checkExpiring(listenStreams.len == 10 and dialStreams.len == 10) + checkExpiring: listenStreams.len == 10 and dialStreams.len == 10 await mplexListen.close() await allFuturesThrowing( @@ -862,7 +862,7 @@ suite "Mplex": for i in 0..9: dialStreams.add((await mplexDial.newStream())) - check await checkExpiring(listenStreams.len == 10 and dialStreams.len == 10) + checkExpiring: listenStreams.len == 10 and dialStreams.len == 10 mplexHandle.cancel() await allFuturesThrowing( @@ -905,7 +905,7 @@ suite "Mplex": for i in 0..9: dialStreams.add((await mplexDial.newStream())) - check await checkExpiring(listenStreams.len == 10 and dialStreams.len == 10) + checkExpiring: listenStreams.len == 10 and dialStreams.len == 10 await conn.close() await allFuturesThrowing( @@ -951,7 +951,7 @@ suite "Mplex": for i in 0..9: dialStreams.add((await mplexDial.newStream())) - check await checkExpiring(listenStreams.len == 10 and dialStreams.len == 10) + checkExpiring: listenStreams.len == 10 and dialStreams.len == 10 await listenConn.closeWithEOF() await allFuturesThrowing( diff --git a/tests/testswitch.nim b/tests/testswitch.nim index 961b6dc3c..1e0db7e15 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -247,14 +247,12 @@ suite "Switch": await switch2.disconnect(switch1.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) - await sleepAsync(1.seconds) - - check: + checkExpiring: startCounts == @[ switch1.connManager.inSema.count, switch1.connManager.outSema.count, @@ -302,7 +300,7 @@ suite "Switch": await switch2.disconnect(switch1.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) @@ -354,7 +352,7 @@ suite "Switch": await switch2.disconnect(switch1.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) @@ -405,7 +403,7 @@ suite "Switch": await switch2.disconnect(switch1.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) @@ -456,7 +454,7 @@ suite "Switch": await switch2.disconnect(switch1.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) @@ -520,8 +518,8 @@ suite "Switch": check not switch2.isConnected(switch1.peerInfo.peerId) check not switch3.isConnected(switch1.peerInfo.peerId) - check await(checkExpiring((not switch1.isConnected(switch2.peerInfo.peerId)))) - check await(checkExpiring((not switch1.isConnected(switch3.peerInfo.peerId)))) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) + checkExpiring: not switch1.isConnected(switch3.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) @@ -554,7 +552,6 @@ suite "Switch": await switches[0].disconnect(peerInfo.peerId) # trigger disconnect of ConnEventKind.Disconnected: check not switches[0].isConnected(peerInfo.peerId) - await sleepAsync(1.millis) done.complete() switches.add(newStandardSwitch( @@ -571,8 +568,6 @@ suite "Switch": await onConnect await done - checkTracker(LPChannelTrackerName) - checkTracker(SecureConnTrackerName) await allFuturesThrowing( switches.mapIt( it.stop() )) @@ -625,42 +620,6 @@ suite "Switch": await allFuturesThrowing( switches.mapIt( it.stop() )) - # TODO: we should be able to test cancellation - # for most of the steps in the upgrade flow - - # this is just a basic test for dials - asyncTest "e2e canceling dial should not leak": - let ma = @[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()] - - let transport = TcpTransport.new(upgrade = Upgrade()) - await transport.start(ma) - - proc acceptHandler() {.async, gcsafe.} = - try: - let conn = await transport.accept() - discard await conn.readLp(100) - await conn.close() - except CatchableError: - discard - - let handlerWait = acceptHandler() - let switch = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) - - await switch.start() - - var peerId = PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).get() - let connectFut = switch.connect(peerId, transport.addrs) - await sleepAsync(500.millis) - connectFut.cancel() - await handlerWait - - checkTracker(LPChannelTrackerName) - checkTracker(SecureConnTrackerName) - checkTracker(ChronosStreamTrackerName) - - await allFuturesThrowing( - transport.stop(), - switch.stop()) - asyncTest "e2e closing remote conn should not leak": let ma = @[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()] @@ -716,7 +675,7 @@ suite "Switch": await allFuturesThrowing(readers) await switch2.stop() #Otherwise this leaks - check await checkExpiring(not switch1.isConnected(switch2.peerInfo.peerId)) + checkExpiring: not switch1.isConnected(switch2.peerInfo.peerId) checkTracker(LPChannelTrackerName) checkTracker(SecureConnTrackerName) From b4f96721af2b7aee59ab6ee437d7e81b223cec0c Mon Sep 17 00:00:00 2001 From: lchenut Date: Sat, 29 Oct 2022 12:12:12 +0200 Subject: [PATCH 16/23] Add a tutorial for the discovery manager (#787) Co-authored-by: Tanguy --- examples/tutorial_5_discovery.nim | 132 ++++++++++++++++++++++++++++++ libp2p.nimble | 2 + libp2p/protocols/rendezvous.nim | 4 +- mkdocs.yml | 1 + 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 examples/tutorial_5_discovery.nim diff --git a/examples/tutorial_5_discovery.nim b/examples/tutorial_5_discovery.nim new file mode 100644 index 000000000..ce02e19df --- /dev/null +++ b/examples/tutorial_5_discovery.nim @@ -0,0 +1,132 @@ +## # Discovery Manager +## +## In the [previous tutorial](tutorial_4_gossipsub.md), we built a custom protocol using [protobuf](https://developers.google.com/protocol-buffers) and +## spread informations (some metrics) on the network using gossipsub. +## For this tutorial, on the other hand, we'll go back on a simple example +## we'll try to discover a specific peers to greet on the network. +## +## First, as usual, we import the dependencies: +import sequtils +import chronos +import stew/byteutils + +import libp2p +import libp2p/protocols/rendezvous +import libp2p/discovery/rendezvousinterface +import libp2p/discovery/discoverymngr + +## We'll not use newStandardSwitch this time as we need the discovery protocol +## [RendezVous](https://github.com/libp2p/specs/blob/master/rendezvous/README.md) to be mounted on the switch using withRendezVous. +## +## Note that other discovery methods such as [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) or [discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) exist. +proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch = + SwitchBuilder.new() + .withRng(newRng()) + .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ]) + .withTcpTransport() + .withYamux() + .withNoise() + .withRendezVous(rdv) + .build() + +# Create a really simple protocol to log one message received then close the stream +const DumbCodec = "/dumb/proto/1.0.0" +type DumbProto = ref object of LPProtocol +proc new(T: typedesc[DumbProto], nodeNumber: int): T = + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + echo "Node", nodeNumber, " received: ", string.fromBytes(await conn.readLp(1024)) + await conn.close() + return T(codecs: @[DumbCodec], handler: handle) + +## ## Bootnodes +## The first time a p2p program is ran, he needs to know how to join +## its network. This is generally done by hard-coding a list of stable +## nodes in the binary, called "bootnodes". These bootnodes are a +## critical part of a p2p network, since they are used by every new +## user to onboard the network. +## +## By using libp2p, we can use any node supporting our discovery protocol +## (rendezvous in this case) as a bootnode. For this example, we'll +## create a bootnode, and then every peer will advertise itself on the +## bootnode, and use it to find other peers +proc main() {.async, gcsafe.} = + let bootNode = createSwitch() + await bootNode.start() + + # Create 5 nodes in the network + var + switches: seq[Switch] = @[] + discManagers: seq[DiscoveryManager] = @[] + + for i in 0..5: + let rdv = RendezVous.new() + # Create a remote future to await at the end of the program + let switch = createSwitch(rdv) + switch.mount(DumbProto.new(i)) + switches.add(switch) + + # A discovery manager is a simple tool, you can set it up by adding discovery + # interfaces (such as RendezVousInterface) then you can use it to advertise + # something on the network or to request something from it. + let dm = DiscoveryManager() + # A RendezVousInterface is a RendezVous protocol wrapped to be usable by the + # DiscoveryManager. + dm.add(RendezVousInterface.new(rdv)) + discManagers.add(dm) + + # We can now start the switch and connect to the bootnode + await switch.start() + await switch.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs) + + # Each nodes of the network will advertise on some topics (EvenGang or OddClub) + dm.advertise(RdvNamespace(if i mod 2 == 0: "EvenGang" else: "OddClub")) + + ## We can now create the newcomer. This peer will connect to the boot node, and use + ## it to discover peers & greet them. + let + rdv = RendezVous.new() + newcomer = createSwitch(rdv) + dm = DiscoveryManager() + await newcomer.start() + await newcomer.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs) + dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds)) + + # Use the discovery manager to find peers on the OddClub topic to greet them + let queryOddClub = dm.request(RdvNamespace("OddClub")) + for _ in 0..2: + let + # getPeer give you a PeerAttribute containing informations about the peer. + res = await queryOddClub.getPeer() + # Here we will use the PeerId and the MultiAddress to greet him + conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec) + await conn.writeLp("Odd Club suuuucks! Even Gang is better!") + # Uh-oh! + await conn.close() + # Wait for the peer to close the stream + await conn.join() + # Queries will run in a loop, so we must stop them when we are done + queryOddClub.stop() + + # Maybe it was because he wanted to join the EvenGang + let queryEvenGang = dm.request(RdvNamespace("EvenGang")) + for _ in 0..2: + let + res = await queryEvenGang.getPeer() + conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec) + await conn.writeLp("Even Gang is sooo laaame! Odd Club rocks!") + # Or maybe not... + await conn.close() + await conn.join() + queryEvenGang.stop() + # What can I say, some people just want to watch the world burn... Anyway + + # Stop all the discovery managers + for d in discManagers: + d.stop() + dm.stop() + + # Stop all the switches + await allFutures(switches.mapIt(it.stop())) + await allFutures(bootNode.stop(), newcomer.stop()) + +waitFor(main()) diff --git a/libp2p.nimble b/libp2p.nimble index 568df0046..1c5f40336 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -91,6 +91,7 @@ task website, "Build the website": tutorialToMd("examples/tutorial_2_customproto.nim") tutorialToMd("examples/tutorial_3_protobuf.nim") tutorialToMd("examples/tutorial_4_gossipsub.nim") + tutorialToMd("examples/tutorial_5_discovery.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -104,6 +105,7 @@ task examples_build, "Build the samples": # These tutorials relies on post 1.4 exception tracking buildSample("tutorial_3_protobuf", true) buildSample("tutorial_4_gossipsub", true) + buildSample("tutorial_5_discovery", true) # pin system # while nimble lockfile diff --git a/libp2p/protocols/rendezvous.nim b/libp2p/protocols/rendezvous.nim index 5460cdc21..1e7639511 100644 --- a/libp2p/protocols/rendezvous.nim +++ b/libp2p/protocols/rendezvous.nim @@ -559,7 +559,9 @@ proc request*(rdv: RendezVous, for (_, r) in s.values(): rdv.save(ns, peer, r, false) - for peer in rdv.peers: + # copy to avoid resizes during the loop + let peers = rdv.peers + for peer in peers: if limit == 0: break if RendezVousCodec notin rdv.switch.peerStore[ProtoBook][peer]: continue try: diff --git a/mkdocs.yml b/mkdocs.yml index c1906f3a3..10d8f05e5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,5 +46,6 @@ nav: - 'Create a custom protocol': tutorial_2_customproto.md - 'Protobuf': tutorial_3_protobuf.md - 'GossipSub': tutorial_4_gossipsub.md + - 'Discovery Manager': tutorial_5_discovery.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' From c39c1cbf6888bc1aea19ef5178fbc995e3814a98 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Sat, 29 Oct 2022 23:26:44 +0200 Subject: [PATCH 17/23] Fix daily CI (#790) --- .github/workflows/multi_nim.yml | 2 +- .pinned | 8 ++++---- libp2p/protocols/pubsub/pubsub.nim | 4 ++-- libp2p/protocols/secure/secure.nim | 1 + libp2p/stream/lpstream.nim | 1 + libp2p/upgrademngrs/upgrade.nim | 1 + 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/multi_nim.yml b/.github/workflows/multi_nim.yml index 4b52a8159..3c7dd608e 100644 --- a/.github/workflows/multi_nim.yml +++ b/.github/workflows/multi_nim.yml @@ -21,7 +21,7 @@ jobs: cpu: amd64 #- os: windows #cpu: i386 - branch: [version-1-2, version-1-4, version-1-6, devel] + branch: [version-1-2, version-1-6, devel] include: - target: os: linux diff --git a/.pinned b/.pinned index 3397fbbad..57b4bb89d 100644 --- a/.pinned +++ b/.pinned @@ -1,7 +1,7 @@ bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a -chronos;https://github.com/status-im/nim-chronos@#9df76c39df254c7ff0cec6dec5c9f345f2819c91 -dnsclient;https://github.com/ba0f3/dnsclient.nim@#6647ca8bd9ffcc13adaecb9cb6453032063967db +chronos;https://github.com/status-im/nim-chronos@#266e2c0ed26b455872bccb3ddbd316815a283659 +dnsclient;https://github.com/ba0f3/dnsclient.nim@#fcd7443634b950eaea574e5eaa00a628ae029823 faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29 httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040 json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 @@ -9,8 +9,8 @@ metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301 nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178 secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466 -stew;https://github.com/status-im/nim-stew@#f2e58ba4c8da65548c824e4fa8732db9739f6505 +stew;https://github.com/status-im/nim-stew@#23da07c9b59c0ba3d4efa7e4e6e2c4121ae5a156 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34 unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db -websock;https://github.com/status-im/nim-websock@#2424f2b215c0546f97d8b147e21544521c7545b0 +websock;https://github.com/status-im/nim-websock@#acbe30e9ca1e51dcbbfe4c552ee6f16c7eede538 zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8 \ No newline at end of file diff --git a/libp2p/protocols/pubsub/pubsub.nim b/libp2p/protocols/pubsub/pubsub.nim index 606744a89..a7d7fc210 100644 --- a/libp2p/protocols/pubsub/pubsub.nim +++ b/libp2p/protocols/pubsub/pubsub.nim @@ -267,11 +267,11 @@ proc updateMetrics*(p: PubSub, rpcMsg: RPCMsg) = method rpcHandler*(p: PubSub, peer: PubSubPeer, - rpcMsg: RPCMsg): Future[void] {.base.} = + rpcMsg: RPCMsg): Future[void] {.base, async.} = ## Handler that must be overridden by concrete implementation raiseAssert "Unimplemented" -method onNewPeer(p: PubSub, peer: PubSubPeer) {.base.} = discard +method onNewPeer(p: PubSub, peer: PubSubPeer) {.base, gcsafe.} = discard method onPubSubPeerEvent*(p: PubSub, peer: PubSubPeer, event: PubSubPeerEvent) {.base, gcsafe.} = # Peer event is raised for the send connection in particular diff --git a/libp2p/protocols/secure/secure.nim b/libp2p/protocols/secure/secure.nim index c187fd2fb..124e1e9b5 100644 --- a/libp2p/protocols/secure/secure.nim +++ b/libp2p/protocols/secure/secure.nim @@ -7,6 +7,7 @@ # This file may not be copied, modified, or distributed except according to # those terms. +{.push gcsafe.} when (NimMajor, NimMinor) < (1, 4): {.push raises: [Defect].} else: diff --git a/libp2p/stream/lpstream.nim b/libp2p/stream/lpstream.nim index fb9401a9a..e2b73278b 100644 --- a/libp2p/stream/lpstream.nim +++ b/libp2p/stream/lpstream.nim @@ -9,6 +9,7 @@ ## Length Prefixed stream implementation +{.push gcsafe.} when (NimMajor, NimMinor) < (1, 4): {.push raises: [Defect].} else: diff --git a/libp2p/upgrademngrs/upgrade.nim b/libp2p/upgrademngrs/upgrade.nim index c5733e658..c087bedba 100644 --- a/libp2p/upgrademngrs/upgrade.nim +++ b/libp2p/upgrademngrs/upgrade.nim @@ -7,6 +7,7 @@ # This file may not be copied, modified, or distributed except according to # those terms. +{.push gcsafe.} when (NimMajor, NimMinor) < (1, 4): {.push raises: [Defect].} else: From c9c2f6acdb0b30477a772f2cf491374d2394c5be Mon Sep 17 00:00:00 2001 From: Tanguy Date: Sun, 30 Oct 2022 07:49:02 +0000 Subject: [PATCH 18/23] Tron example (#775) Co-authored-by: lchenut --- examples/tutorial_6_game.nim | 259 +++++++++++++++++++++++++++++ libp2p.nim | 2 +- libp2p.nimble | 8 +- libp2p/discovery/discoverymngr.nim | 19 +++ libp2p/multiaddress.nim | 3 + libp2p/peerinfo.nim | 23 ++- mkdocs.yml | 3 +- 7 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 examples/tutorial_6_game.nim diff --git a/examples/tutorial_6_game.nim b/examples/tutorial_6_game.nim new file mode 100644 index 000000000..ffbf09a7b --- /dev/null +++ b/examples/tutorial_6_game.nim @@ -0,0 +1,259 @@ +## # Tron example +## +## In this tutorial, we will create a video game based on libp2p, using +## all of the features we talked about in the last tutorials. +## +## We will: +## - Discover peers using the Discovery Manager +## - Use GossipSub to find a play mate +## - Create a custom protocol to play with him +## +## While this may look like a daunting project, it's less than 150 lines of code. +## +## The game will be a simple Tron. We will use [nico](https://github.com/ftsf/nico) +## as a game engine. (you need to run `nimble install nico` to have it available) +## +## ![multiplay](https://user-images.githubusercontent.com/13471753/198852714-b55048e3-f233-4723-900d-2193ad259fe1.gif) +## +## We will start by importing our dependencies and creating our types +import os +import nico, chronos, stew/byteutils, stew/endians2 +import libp2p +import libp2p/protocols/rendezvous +import libp2p/discovery/rendezvousinterface +import libp2p/discovery/discoverymngr + +const + directions = @[(K_UP, 0, -1), (K_LEFT, -1, 0), (K_DOWN, 0, 1), (K_RIGHT, 1, 0)] + mapSize = 32 + tickPeriod = 0.2 + +type + Player = ref object + x, y: int + currentDir, nextDir: int + lost: bool + color: int + + Game = ref object + gameMap: array[mapSize * mapSize, int] + tickTime: float + localPlayer, remotePlayer: Player + peerFound: Future[Connection] + hasCandidate: bool + tickFinished: Future[int] + + GameProto = ref object of LPProtocol + +proc new(_: type[Game]): Game = + # Default state of a game + result = Game( + tickTime: -3.0, # 3 seconds of "warm-up" time + localPlayer: Player(x: 4, y: 16, currentDir: 3, nextDir: 3, color: 8), + remotePlayer: Player(x: 27, y: 16, currentDir: 1, nextDir: 1, color: 12), + peerFound: newFuture[Connection]() + ) + for pos in 0 .. result.gameMap.high: + if pos mod mapSize in [0, mapSize - 1] or pos div mapSize in [0, mapSize - 1]: + result.gameMap[pos] = 7 + +## ## Game Logic +## The networking during the game will work like this: +## +## * Each player will have `tickPeriod` (0.1) seconds to choose +## a direction that he wants to go to (default to current direction) +## * After `tickPeriod`, we will send our choosen direction to the peer, +## and wait for his direction +## * Once we have both direction, we will "tick" the game, and restart the +## loop, as long as both player are alive. +## +## This is a very simplistic scheme, but creating proper networking for +## video games is an [art](https://developer.valvesoftware.com/wiki/Latency_Compensating_Methods_in_Client/Server_In-game_Protocol_Design_and_Optimization) +## +## The main drawback of this scheme is that the more ping you have with +## the peer, the slower the game will run. Or invertedly, the less ping you +## have, the faster it runs! +proc update(g: Game, dt: float32) = + # Will be called at each frame of the game. + # + # Because both Nico and Chronos have a main loop, + # they must share the control of the main thread. + # This is a hacky way to make this happen + waitFor(sleepAsync(1.milliseconds)) + # Don't do anything if we are still waiting for an opponent + if not(g.peerFound.finished()) or isNil(g.tickFinished): return + g.tickTime += dt + + # Update the wanted direction, making sure we can't go backward + for i in 0 .. directions.high: + if i != (g.localPlayer.currentDir + 2 mod 4) and keyp(directions[i][0]): + g.localPlayer.nextDir = i + + if g.tickTime > tickPeriod and not g.tickFinished.finished(): + # We choosen our next direction, let the networking know + g.localPlayer.currentDir = g.localPlayer.nextDir + g.tickFinished.complete(g.localPlayer.currentDir) + +proc tick(g: Game, p: Player) = + # Move player and check if he lost + p.x += directions[p.currentDir][1] + p.y += directions[p.currentDir][2] + if g.gameMap[p.y * mapSize + p.x] != 0: p.lost = true + g.gameMap[p.y * mapSize + p.x] = p.color + +proc mainLoop(g: Game, peer: Connection) {.async.} = + while not (g.localPlayer.lost or g.remotePlayer.lost): + if g.tickTime > 0.0: + g.tickTime = 0 + g.tickFinished = newFuture[int]() + + # Wait for a choosen direction + let dir = await g.tickFinished + # Send it + await peer.writeLp(toBytes(uint32(dir))) + + # Get the one from the peer + g.remotePlayer.currentDir = int uint32.fromBytes(await peer.readLp(8)) + # Tick the players & restart + g.tick(g.remotePlayer) + g.tick(g.localPlayer) + +## We'll draw the map & put some texts when necessary: +proc draw(g: Game) = + for pos, color in g.gameMap: + setColor(color) + boxFill(pos mod 32 * 4, pos div 32 * 4, 4, 4) + let text = if not(g.peerFound.finished()): "Matchmaking.." + elif g.tickTime < -1.5: "Welcome to Etron" + elif g.tickTime < 0.0: "- " & $(int(abs(g.tickTime) / 0.5) + 1) & " -" + elif g.remotePlayer.lost and g.localPlayer.lost: "DEUCE" + elif g.localPlayer.lost: "YOU LOOSE" + elif g.remotePlayer.lost: "YOU WON" + else: "" + printc(text, screenWidth div 2, screenHeight div 2) + + +## ## Matchmaking +## To find an opponent, we will broadcast our address on a +## GossipSub topic, and wait for someone to connect to us. +## We will also listen to that topic, and connect to anyone +## broadcasting his address. +## +## If we are looking for a game, we'll send `ok` to let the +## peer know that we are available, check that he is also available, +## and launch the game. +proc new(T: typedesc[GameProto], g: Game): T = + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + defer: await conn.closeWithEof() + if g.peerFound.finished or g.hasCandidate: + await conn.close() + return + g.hasCandidate = true + await conn.writeLp("ok") + if "ok" != string.fromBytes(await conn.readLp(1024)): + g.hasCandidate = false + return + g.peerFound.complete(conn) + # The handler of a protocol must wait for the stream to + # be finished before returning + await conn.join() + return T(codecs: @["/tron/1.0.0"], handler: handle) + +proc networking(g: Game) {.async.} = + # Create our switch, similar to the GossipSub example and + # the Discovery examples combined + let + rdv = RendezVous.new() + switch = SwitchBuilder.new() + .withRng(newRng()) + .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ]) + .withTcpTransport() + .withYamux() + .withNoise() + .withRendezVous(rdv) + .build() + dm = DiscoveryManager() + gameProto = GameProto.new(g) + gossip = GossipSub.init( + switch = switch, + triggerSelf = false) + dm.add(RendezVousInterface.new(rdv)) + + switch.mount(gossip) + switch.mount(gameProto) + + gossip.subscribe( + "/tron/matchmaking", + proc (topic: string, data: seq[byte]) {.async.} = + # If we are still looking for an opponent, + # try to match anyone broadcasting it's address + if g.peerFound.finished or g.hasCandidate: return + g.hasCandidate = true + + try: + let + (peerId, multiAddress) = parseFullAddress(data).tryGet() + stream = await switch.dial(peerId, @[multiAddress], gameProto.codec) + + await stream.writeLp("ok") + if (await stream.readLp(10)) != "ok".toBytes: + g.hasCandidate = false + return + g.peerFound.complete(stream) + # We are "player 2" + swap(g.localPlayer, g.remotePlayer) + except CatchableError as exc: + discard + ) + + await switch.start() + defer: await switch.stop() + + # As explained in the last tutorial, we need a bootnode to be able + # to find peers. We could use any libp2p running rendezvous (or any + # node running tron). We will take it's MultiAddress from the command + # line parameters + if paramCount() > 0: + let (peerId, multiAddress) = paramStr(1).parseFullAddress().tryGet() + await switch.connect(peerId, @[multiAddress]) + else: + echo "No bootnode provided, listening on: ", switch.peerInfo.fullAddrs.tryGet() + + # Discover peers from the bootnode, and connect to them + dm.advertise(RdvNamespace("tron")) + let discoveryQuery = dm.request(RdvNamespace("tron")) + discoveryQuery.forEach: + try: + await switch.connect(peer[PeerId], peer.getAll(MultiAddress)) + except CatchableError as exc: + echo "Failed to dial a peer: ", exc.msg + + # We will try to publish our address multiple times, in case + # it takes time to establish connections with other GossipSub peers + var published = false + while not published: + await sleepAsync(500.milliseconds) + for fullAddr in switch.peerInfo.fullAddrs.tryGet(): + if (await gossip.publish("/tron/matchmaking", fullAddr.bytes)) == 0: + published = false + break + published = true + + discoveryQuery.stop() + + # We now wait for someone to connect to us (or for us to connect to someone) + let peerConn = await g.peerFound + defer: await peerConn.closeWithEof() + + await g.mainLoop(peerConn) + +let + game = Game.new() + netFut = networking(game) +nico.init("Status", "Tron") +nico.createWindow("Tron", mapSize * 4, mapSize * 4, 4, false) +nico.run(proc = discard, proc(dt: float32) = game.update(dt), proc = game.draw()) +waitFor(netFut.cancelAndWait()) + +## And that's it! If you want to run this code locally, the simplest way is to use the +## first node as a boot node for the second one. But you can also use any rendezvous node diff --git a/libp2p.nim b/libp2p.nim index 46f802320..8d44aae6f 100644 --- a/libp2p.nim +++ b/libp2p.nim @@ -17,7 +17,7 @@ when defined(nimdoc): ## stay backward compatible during the Major version, whereas private ones can ## change at each new Minor version. ## - ## If you're new to nim-libp2p, you can find a tutorial `here`_ + ## If you're new to nim-libp2p, you can find a tutorial `here`_ ## that can help you get started. # Import stuff for doc diff --git a/libp2p.nimble b/libp2p.nimble index 1c5f40336..c0aa3c513 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -31,8 +31,8 @@ proc runTest(filename: string, verify: bool = true, sign: bool = true, exec excstr & " -r " & " tests/" & filename rmFile "tests/" & filename.toExe -proc buildSample(filename: string, run = false) = - var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " +proc buildSample(filename: string, run = false, extraFlags = "") = + var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " & extraFlags excstr.add(" examples/" & filename) exec excstr if run: @@ -92,6 +92,7 @@ task website, "Build the website": tutorialToMd("examples/tutorial_3_protobuf.nim") tutorialToMd("examples/tutorial_4_gossipsub.nim") tutorialToMd("examples/tutorial_5_discovery.nim") + tutorialToMd("examples/tutorial_6_game.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -106,6 +107,9 @@ task examples_build, "Build the samples": buildSample("tutorial_3_protobuf", true) buildSample("tutorial_4_gossipsub", true) buildSample("tutorial_5_discovery", true) + # Nico doesn't work in 1.2 + exec "nimble install -y nico" + buildSample("tutorial_6_game", false, "--styleCheck:off") # pin system # while nimble lockfile diff --git a/libp2p/discovery/discoverymngr.nim b/libp2p/discovery/discoverymngr.nim index 6c1854361..140f5335e 100644 --- a/libp2p/discovery/discoverymngr.nim +++ b/libp2p/discovery/discoverymngr.nim @@ -89,10 +89,12 @@ method advertise*(self: DiscoveryInterface) {.async, base.} = type DiscoveryError* = object of LPError + DiscoveryFinished* = object of LPError DiscoveryQuery* = ref object attr: PeerAttributes peers: AsyncQueue[PeerAttributes] + finished: bool futs: seq[Future[void]] DiscoveryManager* = ref object @@ -137,7 +139,22 @@ proc advertise*[T](dm: DiscoveryManager, value: T) = pa.add(value) dm.advertise(pa) +template forEach*(query: DiscoveryQuery, code: untyped) = + ## Will execute `code` for each discovered peer. The + ## peer attritubtes are available through the variable + ## `peer` + + proc forEachInternal(q: DiscoveryQuery) {.async.} = + while true: + let peer {.inject.} = + try: await q.getPeer() + except DiscoveryFinished: return + code + + asyncSpawn forEachInternal(query) + proc stop*(query: DiscoveryQuery) = + query.finished = true for r in query.futs: if not r.finished(): r.cancel() @@ -158,6 +175,8 @@ proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} = raise exc if not finished(getter): + if query.finished: + raise newException(DiscoveryFinished, "Discovery query stopped") # discovery loops only finish when they don't handle the query raise newException(DiscoveryError, "Unable to find any peer matching this request") return await getter diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index d7ef6e9cc..055d8b138 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -1084,6 +1084,9 @@ proc `$`*(pat: MaPattern): string = elif pat.operator == Eq: result = $pat.value +proc bytes*(value: MultiAddress): seq[byte] = + value.data.buffer + proc write*(pb: var ProtoBuffer, field: int, value: MultiAddress) {.inline.} = write(pb, field, value.data.buffer) diff --git a/libp2p/peerinfo.nim b/libp2p/peerinfo.nim index 39a6ad63c..12aeb47c5 100644 --- a/libp2p/peerinfo.nim +++ b/libp2p/peerinfo.nim @@ -15,7 +15,7 @@ else: import std/[options, sequtils] import pkg/[chronos, chronicles, stew/results] -import peerid, multiaddress, crypto/crypto, routing_record, errors, utility +import peerid, multiaddress, multicodec, crypto/crypto, routing_record, errors, utility export peerid, multiaddress, crypto, routing_record, errors, results @@ -69,6 +69,27 @@ proc update*(p: PeerInfo) {.async.} = proc addrs*(p: PeerInfo): seq[MultiAddress] = p.addrs +proc fullAddrs*(p: PeerInfo): MaResult[seq[MultiAddress]] = + let peerIdPart = ? MultiAddress.init(multiCodec("p2p"), p.peerId.data) + var res: seq[MultiAddress] + for address in p.addrs: + res.add(? concat(address, peerIdPart)) + ok(res) + +proc parseFullAddress*(ma: MultiAddress): MaResult[(PeerId, MultiAddress)] = + let p2pPart = ? ma[^1] + if ? p2pPart.protoCode != multiCodec("p2p"): + return err("Missing p2p part from multiaddress!") + + let res = ( + ? PeerId.init(? p2pPart.protoArgument()).orErr("invalid peerid"), + ? ma[0 .. ^2] + ) + ok(res) + +proc parseFullAddress*(ma: string | seq[byte]): MaResult[(PeerId, MultiAddress)] = + parseFullAddress(? MultiAddress.init(ma)) + proc new*( p: typedesc[PeerInfo], key: PrivateKey, diff --git a/mkdocs.yml b/mkdocs.yml index 10d8f05e5..e9cdaacd6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,12 +40,13 @@ theme: name: Switch to light mode nav: - - Introduction: README.md - Tutorials: + - 'Introduction': README.md - 'Simple connection': tutorial_1_connect.md - 'Create a custom protocol': tutorial_2_customproto.md - 'Protobuf': tutorial_3_protobuf.md - 'GossipSub': tutorial_4_gossipsub.md - 'Discovery Manager': tutorial_5_discovery.md + - 'Game': tutorial_6_game.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' From a3e9d1ed80c048cd5abc839cbe0863cefcedc702 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Sun, 30 Oct 2022 12:18:04 +0000 Subject: [PATCH 19/23] Version 1.0.0 (#785) --- README.md | 30 +++++++++++++++++++----------- libp2p.nimble | 2 +- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e8ec029d8..88fc4f9c1 100644 --- a/README.md +++ b/README.md @@ -98,20 +98,31 @@ nim-libp2p is used by: - [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application - (open a pull request if you want to be included here) -## Development -**Clone and Install dependencies:** +## Stability +nim-libp2p has been used in production for over a year in high-stake scenarios, so its core is considered stable. +Some modules are more recent and less stable. +The versioning follows [semver](https://semver.org/), with some additions: +- Some of libp2p procedures are marked as `.public.`, they will remain compatible during each `MAJOR` version +- The rest of the procedures are considered internal, and can change at any `MINOR` version (but remain compatible for each new `PATCH`) + +We aim to be compatible at all time with at least 2 Nim `MINOR` versions, currently `1.2 & 1.6` + +## Development +Clone and Install dependencies: ```sh git clone https://github.com/status-im/nim-libp2p cd nim-libp2p -nimble install +nimble install -dy ``` -**Run unit tests** +Run unit tests: ```sh # run all the unit tests nimble test ``` +This requires the go daemon to be available. To only run native tests, use `nimble testnative`. +Or use `nimble tasks` to show all available tasks. ### Contribute @@ -125,22 +136,19 @@ The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-st ### Core Developers [@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Tanguy](https://github.com/Menduist), [Zahary Karadjov](https://github.com/zah) -### Tips and tricks - -**enable expensive metrics:** +### Compile time flags +Enable expensive metrics (ie, metrics with per-peer cardinality): ```bash nim c -d:libp2p_expensive_metrics some_file.nim ``` -**use identify metrics** - +Set list of known libp2p agents for metrics: ```bash nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku some_file.nim ``` -**specify gossipsub specific topics to measure** - +Specify gossipsub specific topics to measure in the metrics: ```bash nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim ``` diff --git a/libp2p.nimble b/libp2p.nimble index c0aa3c513..ff4f8bd65 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -1,7 +1,7 @@ mode = ScriptMode.Verbose packageName = "libp2p" -version = "0.0.2" +version = "1.0.0" author = "Status Research & Development GmbH" description = "LibP2P implementation" license = "MIT" From f95eda8bf6f878b2c9396386f730ff80acd2f278 Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Fri, 4 Nov 2022 11:19:27 +0100 Subject: [PATCH 20/23] Add libp2p label to chronicles log topics (#794) --- libp2p/protocols/pubsub/gossipsub/behavior.nim | 3 +++ libp2p/protocols/pubsub/gossipsub/scoring.nim | 3 +++ libp2p/protocols/pubsub/rpc/protobuf.nim | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libp2p/protocols/pubsub/gossipsub/behavior.nim b/libp2p/protocols/pubsub/gossipsub/behavior.nim index b3b1f4cad..b2c0255e4 100644 --- a/libp2p/protocols/pubsub/gossipsub/behavior.nim +++ b/libp2p/protocols/pubsub/gossipsub/behavior.nim @@ -19,6 +19,9 @@ import ".."/[pubsubpeer, peertable, timedcache, mcache, floodsub, pubsub] import "../rpc"/[messages] import "../../.."/[peerid, multiaddress, utility, switch, routing_record, signed_envelope, utils/heartbeat] +logScope: + topics = "libp2p gossipsub" + declareGauge(libp2p_gossipsub_cache_window_size, "the number of messages in the cache") declareGauge(libp2p_gossipsub_peers_per_topic_mesh, "gossipsub peers per topic in mesh", labels = ["topic"]) declareGauge(libp2p_gossipsub_peers_per_topic_fanout, "gossipsub peers per topic in fanout", labels = ["topic"]) diff --git a/libp2p/protocols/pubsub/gossipsub/scoring.nim b/libp2p/protocols/pubsub/gossipsub/scoring.nim index 801134d2f..93a4fb3c8 100644 --- a/libp2p/protocols/pubsub/gossipsub/scoring.nim +++ b/libp2p/protocols/pubsub/gossipsub/scoring.nim @@ -18,6 +18,9 @@ import "."/[types] import ".."/[pubsubpeer] import "../../.."/[peerid, multiaddress, utility, switch, utils/heartbeat] +logScope: + topics = "libp2p gossipsub" + declareGauge(libp2p_gossipsub_peers_scores, "the scores of the peers in gossipsub", labels = ["agent"]) declareCounter(libp2p_gossipsub_bad_score_disconnection, "the number of peers disconnected by gossipsub", labels = ["agent"]) declareGauge(libp2p_gossipsub_peers_score_firstMessageDeliveries, "Detailed gossipsub scoring metric", labels = ["agent"]) diff --git a/libp2p/protocols/pubsub/rpc/protobuf.nim b/libp2p/protocols/pubsub/rpc/protobuf.nim index 1b4cd2268..b2d6a0e2f 100644 --- a/libp2p/protocols/pubsub/rpc/protobuf.nim +++ b/libp2p/protocols/pubsub/rpc/protobuf.nim @@ -22,7 +22,7 @@ import messages, logScope: - topics = "pubsubprotobuf" + topics = "libp2p pubsubprotobuf" when defined(libp2p_protobuf_metrics): import metrics From d9305bda84d43858b0102b3bb35ce74f5ada2c73 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Fri, 4 Nov 2022 19:20:23 +0100 Subject: [PATCH 21/23] Add Tor Transport support (#765) --- libp2p/multiaddress.nim | 2 + libp2p/stream/lpstream.nim | 2 +- libp2p/transports/tortransport.nim | 281 +++++++++++++++++++++++++++++ tests/commontransport.nim | 55 +++--- tests/stubs.nim | 86 +++++++++ tests/testnative.nim | 1 + tests/testtcptransport.nim | 5 +- tests/testtortransport.nim | 143 +++++++++++++++ tests/testwstransport.nim | 15 +- 9 files changed, 552 insertions(+), 38 deletions(-) create mode 100644 libp2p/transports/tortransport.nim create mode 100644 tests/stubs.nim create mode 100644 tests/testtortransport.nim diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index 055d8b138..211954d95 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -470,6 +470,8 @@ const WS* = mapAnd(TCP, mapEq("ws")) WSS* = mapAnd(TCP, mapEq("wss")) WebSockets* = mapOr(WS, WSS) + Onion3* = mapEq("onion3") + TcpOnion3* = mapAnd(TCP, Onion3) Unreliable* = mapOr(UDP) diff --git a/libp2p/stream/lpstream.nim b/libp2p/stream/lpstream.nim index e2b73278b..e604baa23 100644 --- a/libp2p/stream/lpstream.nim +++ b/libp2p/stream/lpstream.nim @@ -80,7 +80,7 @@ type opened*: uint64 closed*: uint64 -proc setupStreamTracker(name: string): StreamTracker = +proc setupStreamTracker*(name: string): StreamTracker = let tracker = new StreamTracker proc dumpTracking(): string {.gcsafe.} = diff --git a/libp2p/transports/tortransport.nim b/libp2p/transports/tortransport.nim new file mode 100644 index 000000000..e978be90c --- /dev/null +++ b/libp2p/transports/tortransport.nim @@ -0,0 +1,281 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +## Tor transport implementation + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/strformat +import chronos, chronicles, strutils +import stew/[byteutils, endians2, results, objects] +import ../multicodec +import transport, + tcptransport, + ../switch, + ../builders, + ../stream/[lpstream, connection, chronosstream], + ../multiaddress, + ../upgrademngrs/upgrade + +const + IPTcp = mapAnd(IP, mapEq("tcp")) + IPv4Tcp = mapAnd(IP4, mapEq("tcp")) + IPv6Tcp = mapAnd(IP6, mapEq("tcp")) + DnsTcp = mapAnd(DNSANY, mapEq("tcp")) + + Socks5ProtocolVersion = byte(5) + NMethods = byte(1) + +type + TorTransport* = ref object of Transport + transportAddress: TransportAddress + tcpTransport: TcpTransport + + Socks5AuthMethod* {.pure.} = enum + NoAuth = 0 + GSSAPI = 1 + UsernamePassword = 2 + NoAcceptableMethod = 0xff + + Socks5RequestCommand* {.pure.} = enum + Connect = 1, Bind = 2, UdpAssoc = 3 + + Socks5AddressType* {.pure.} = enum + IPv4 = 1, FQDN = 3, IPv6 = 4 + + Socks5ReplyType* {.pure.} = enum + Succeeded = (0, "Succeeded"), ServerFailure = (1, "Server Failure"), + ConnectionNotAllowed = (2, "Connection Not Allowed"), NetworkUnreachable = (3, "Network Unreachable"), + HostUnreachable = (4, "Host Unreachable"), ConnectionRefused = (5, "Connection Refused"), + TtlExpired = (6, "Ttl Expired"), CommandNotSupported = (7, "Command Not Supported"), + AddressTypeNotSupported = (8, "Address Type Not Supported") + + TransportStartError* = object of transport.TransportError + + Socks5Error* = object of CatchableError + Socks5AuthFailedError* = object of Socks5Error + Socks5VersionError* = object of Socks5Error + Socks5ServerReplyError* = object of Socks5Error + +proc new*( + T: typedesc[TorTransport], + transportAddress: TransportAddress, + flags: set[ServerFlags] = {}, + upgrade: Upgrade): T {.public.} = + ## Creates a Tor transport + + T( + transportAddress: transportAddress, + upgrader: upgrade, + tcpTransport: TcpTransport.new(flags, upgrade)) + +proc handlesDial(address: MultiAddress): bool {.gcsafe.} = + return Onion3.match(address) or TCP.match(address) or DNSANY.match(address) + +proc handlesStart(address: MultiAddress): bool {.gcsafe.} = + return TcpOnion3.match(address) + +proc connectToTorServer( + transportAddress: TransportAddress): Future[StreamTransport] {.async, gcsafe.} = + let transp = await connect(transportAddress) + try: + discard await transp.write(@[Socks5ProtocolVersion, NMethods, Socks5AuthMethod.NoAuth.byte]) + let + serverReply = await transp.read(2) + socks5ProtocolVersion = serverReply[0] + serverSelectedMethod = serverReply[1] + if socks5ProtocolVersion != Socks5ProtocolVersion: + raise newException(Socks5VersionError, "Unsupported socks version") + if serverSelectedMethod != Socks5AuthMethod.NoAuth.byte: + raise newException(Socks5AuthFailedError, "Unsupported auth method") + return transp + except CatchableError as err: + await transp.closeWait() + raise err + +proc readServerReply(transp: StreamTransport) {.async, gcsafe.} = + ## The specification for this code is defined on + ## [link text](https://www.rfc-editor.org/rfc/rfc1928#section-5) + ## and [link text](https://www.rfc-editor.org/rfc/rfc1928#section-6). + let + portNumOctets = 2 + ipV4NumOctets = 4 + ipV6NumOctets = 16 + firstFourOctets = await transp.read(4) + socks5ProtocolVersion = firstFourOctets[0] + serverReply = firstFourOctets[1] + if socks5ProtocolVersion != Socks5ProtocolVersion: + raise newException(Socks5VersionError, "Unsupported socks version") + if serverReply != Socks5ReplyType.Succeeded.byte: + var socks5ReplyType: Socks5ReplyType + if socks5ReplyType.checkedEnumAssign(serverReply): + raise newException(Socks5ServerReplyError, fmt"Server reply error: {socks5ReplyType}") + else: + raise newException(LPError, fmt"Unexpected server reply: {serverReply}") + let atyp = firstFourOctets[3] + case atyp: + of Socks5AddressType.IPv4.byte: + discard await transp.read(ipV4NumOctets + portNumOctets) + of Socks5AddressType.FQDN.byte: + let fqdnNumOctets = await transp.read(1) + discard await transp.read(int(uint8.fromBytes(fqdnNumOctets)) + portNumOctets) + of Socks5AddressType.IPv6.byte: + discard await transp.read(ipV6NumOctets + portNumOctets) + else: + raise newException(LPError, "Address not supported") + +proc parseOnion3(address: MultiAddress): (byte, seq[byte], seq[byte]) {.raises: [Defect, LPError, ValueError].} = + var addressArray = ($address).split('/') + if addressArray.len < 2: raise newException(LPError, fmt"Onion address not supported {address}") + addressArray = addressArray[2].split(':') + if addressArray.len == 0: raise newException(LPError, fmt"Onion address not supported {address}") + let + addressStr = addressArray[0] & ".onion" + dstAddr = @(uint8(addressStr.len).toBytes()) & addressStr.toBytes() + dstPort = address.data.buffer[37..38] + return (Socks5AddressType.FQDN.byte, dstAddr, dstPort) + +proc parseIpTcp(address: MultiAddress): (byte, seq[byte], seq[byte]) {.raises: [Defect, LPError, ValueError].} = + let (codec, atyp) = + if IPv4Tcp.match(address): + (multiCodec("ip4"), Socks5AddressType.IPv4.byte) + elif IPv6Tcp.match(address): + (multiCodec("ip6"), Socks5AddressType.IPv6.byte) + else: + raise newException(LPError, fmt"IP address not supported {address}") + let + dstAddr = address[codec].get().protoArgument().get() + dstPort = address[multiCodec("tcp")].get().protoArgument().get() + (atyp, dstAddr, dstPort) + +proc parseDnsTcp(address: MultiAddress): (byte, seq[byte], seq[byte]) = + let + dnsAddress = address[multiCodec("dns")].get().protoArgument().get() + dstAddr = @(uint8(dnsAddress.len).toBytes()) & dnsAddress + dstPort = address[multiCodec("tcp")].get().protoArgument().get() + (Socks5AddressType.FQDN.byte, dstAddr, dstPort) + +proc dialPeer( + transp: StreamTransport, address: MultiAddress) {.async, gcsafe.} = + let (atyp, dstAddr, dstPort) = + if Onion3.match(address): + parseOnion3(address) + elif IPTcp.match(address): + parseIpTcp(address) + elif DnsTcp.match(address): + parseDnsTcp(address) + else: + raise newException(LPError, fmt"Address not supported: {address}") + + let reserved = byte(0) + let request = @[ + Socks5ProtocolVersion, + Socks5RequestCommand.Connect.byte, + reserved, + atyp] & dstAddr & dstPort + discard await transp.write(request) + await readServerReply(transp) + +method dial*( + self: TorTransport, + hostname: string, + address: MultiAddress): Future[Connection] {.async, gcsafe.} = + ## dial a peer + ## + if not handlesDial(address): + raise newException(LPError, fmt"Address not supported: {address}") + trace "Dialing remote peer", address = $address + let transp = await connectToTorServer(self.transportAddress) + + try: + await dialPeer(transp, address) + return await self.tcpTransport.connHandler(transp, Opt.none(MultiAddress), Direction.Out) + except CatchableError as err: + await transp.closeWait() + raise err + +method start*( + self: TorTransport, + addrs: seq[MultiAddress]) {.async.} = + ## listen on the transport + ## + + var listenAddrs: seq[MultiAddress] + var onion3Addrs: seq[MultiAddress] + for i, ma in addrs: + if not handlesStart(ma): + warn "Invalid address detected, skipping!", address = ma + continue + + let listenAddress = ma[0..1].get() + listenAddrs.add(listenAddress) + let onion3 = ma[multiCodec("onion3")].get() + onion3Addrs.add(onion3) + + if len(listenAddrs) != 0 and len(onion3Addrs) != 0: + await procCall Transport(self).start(onion3Addrs) + await self.tcpTransport.start(listenAddrs) + else: + raise newException(TransportStartError, "Tor Transport couldn't start, no supported addr was provided.") + +method accept*(self: TorTransport): Future[Connection] {.async, gcsafe.} = + ## accept a new Tor connection + ## + let conn = await self.tcpTransport.accept() + conn.observedAddr = Opt.none(MultiAddress) + return conn + +method stop*(self: TorTransport) {.async, gcsafe.} = + ## stop the transport + ## + await procCall Transport(self).stop() # call base + await self.tcpTransport.stop() + +method handles*(t: TorTransport, address: MultiAddress): bool {.gcsafe.} = + if procCall Transport(t).handles(address): + return handlesDial(address) or handlesStart(address) + +type + TorSwitch* = ref object of Switch + +proc new*( + T: typedesc[TorSwitch], + torServer: TransportAddress, + rng: ref HmacDrbgContext, + addresses: seq[MultiAddress] = @[], + flags: set[ServerFlags] = {}): TorSwitch + {.raises: [LPError, Defect], public.} = + var builder = SwitchBuilder.new() + .withRng(rng) + .withTransport(proc(upgr: Upgrade): Transport = TorTransport.new(torServer, flags, upgr)) + if addresses.len != 0: + builder = builder.withAddresses(addresses) + let switch = builder.withMplex() + .withNoise() + .build() + let torSwitch = T( + peerInfo: switch.peerInfo, + ms: switch.ms, + transports: switch.transports, + connManager: switch.connManager, + peerStore: switch.peerStore, + dialer: Dialer.new(switch.peerInfo.peerId, switch.connManager, switch.transports, switch.ms, nil), + nameResolver: nil) + + torSwitch.connManager.peerStore = switch.peerStore + return torSwitch + +method addTransport*(s: TorSwitch, t: Transport) = + doAssert(false, "not implemented!") + +method getTorTransport*(s: TorSwitch): Transport {.base.} = + return s.transports[0] diff --git a/tests/commontransport.nim b/tests/commontransport.nim index af29add61..b46479b54 100644 --- a/tests/commontransport.nim +++ b/tests/commontransport.nim @@ -13,25 +13,24 @@ import ./helpers type TransportProvider* = proc(): Transport {.gcsafe, raises: [Defect].} -proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = - suite name & " common tests": - teardown: - checkTrackers() +template commonTransportTest*(prov: TransportProvider, ma1: string, ma2: string = "") = + block: + let transpProvider = prov asyncTest "can handle local address": - let ma = @[MultiAddress.init(ma).tryGet()] - let transport1 = prov() + let ma = @[MultiAddress.init(ma1).tryGet()] + let transport1 = transpProvider() await transport1.start(ma) check transport1.handles(transport1.addrs[0]) await transport1.stop() asyncTest "e2e: handle observedAddr": - let ma = @[MultiAddress.init(ma).tryGet()] + let ma = @[MultiAddress.init(ma1).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(ma) - let transport2 = prov() + let transport2 = transpProvider() proc acceptHandler() {.async, gcsafe.} = let conn = await transport1.accept() @@ -56,9 +55,9 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = await handlerWait.wait(1.seconds) # when no issues will not wait that long! asyncTest "e2e: handle write": - let ma = @[MultiAddress.init(ma).tryGet()] + let ma = @[MultiAddress.init(ma1).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(ma) proc acceptHandler() {.async, gcsafe.} = @@ -68,7 +67,7 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = let handlerWait = acceptHandler() - let transport2 = prov() + let transport2 = transpProvider() let conn = await transport2.dial(transport1.addrs[0]) var msg = newSeq[byte](6) await conn.readExactly(addr msg[0], 6) @@ -84,8 +83,8 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = await handlerWait.wait(1.seconds) # when no issues will not wait that long! asyncTest "e2e: handle read": - let ma = @[MultiAddress.init(ma).tryGet()] - let transport1 = prov() + let ma = @[MultiAddress.init(ma1).tryGet()] + let transport1 = transpProvider() await transport1.start(ma) proc acceptHandler() {.async, gcsafe.} = @@ -97,7 +96,7 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = let handlerWait = acceptHandler() - let transport2 = prov() + let transport2 = transpProvider() let conn = await transport2.dial(transport1.addrs[0]) await conn.write("Hello!") @@ -110,12 +109,12 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = transport2.stop())) asyncTest "e2e: handle dial cancellation": - let ma = @[MultiAddress.init(ma).tryGet()] + let ma = @[MultiAddress.init(ma1).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(ma) - let transport2 = prov() + let transport2 = transpProvider() let cancellation = transport2.dial(transport1.addrs[0]) await cancellation.cancelAndWait() @@ -127,9 +126,9 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = transport2.stop())) asyncTest "e2e: handle accept cancellation": - let ma = @[MultiAddress.init(ma).tryGet()] + let ma = @[MultiAddress.init(ma1).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(ma) let acceptHandler = transport1.accept() @@ -143,11 +142,11 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = # this randomly locks the Windows CI job skip() return - let addrs = @[MultiAddress.init(ma).tryGet(), - MultiAddress.init(ma).tryGet()] + let addrs = @[MultiAddress.init(ma1).tryGet(), + MultiAddress.init(if ma2 == "": ma1 else: ma2).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(addrs) proc acceptHandler() {.async, gcsafe.} = @@ -192,12 +191,12 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = await transport1.stop() asyncTest "e2e: stopping transport kills connections": - let ma = @[MultiAddress.init(ma).tryGet()] + let ma = @[MultiAddress.init(ma1).tryGet()] - let transport1 = prov() + let transport1 = transpProvider() await transport1.start(ma) - let transport2 = prov() + let transport2 = transpProvider() let acceptHandler = transport1.accept() let conn = await transport2.dial(transport1.addrs[0]) @@ -212,8 +211,8 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = check conn.closed() asyncTest "read or write on closed connection": - let ma = @[MultiAddress.init(ma).tryGet()] - let transport1 = prov() + let ma = @[MultiAddress.init(ma1).tryGet()] + let transport1 = transpProvider() await transport1.start(ma) proc acceptHandler() {.async, gcsafe.} = diff --git a/tests/stubs.nim b/tests/stubs.nim new file mode 100644 index 000000000..aae661de4 --- /dev/null +++ b/tests/stubs.nim @@ -0,0 +1,86 @@ +{.used.} + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import tables +import chronos, stew/[byteutils, endians2, shims/net] +import ../libp2p/[stream/connection, + protocols/connectivity/relay/utils, + transports/tcptransport, + transports/tortransport, + upgrademngrs/upgrade, + multiaddress, + errors, + builders] + +type + TorServerStub* = ref object of RootObj + tcpTransport: TcpTransport + addrTable: Table[string, string] + +proc new*( + T: typedesc[TorServerStub]): T {.public.} = + + T( + tcpTransport: TcpTransport.new(flags = {ReuseAddr}, upgrade = Upgrade()), + addrTable: initTable[string, string]()) + +proc registerAddr*(self: TorServerStub, key: string, val: string) = + self.addrTable[key] = val + +proc start*(self: TorServerStub, address: TransportAddress) {.async.} = + let ma = @[MultiAddress.init(address).tryGet()] + + await self.tcpTransport.start(ma) + + var msg = newSeq[byte](3) + while self.tcpTransport.running: + let connSrc = await self.tcpTransport.accept() + await connSrc.readExactly(addr msg[0], 3) + + await connSrc.write(@[05'u8, 00]) + + msg = newSeq[byte](4) + await connSrc.readExactly(addr msg[0], 4) + let atyp = msg[3] + let address = case atyp: + of Socks5AddressType.IPv4.byte: + let n = 4 + 2 # +2 bytes for the port + msg = newSeq[byte](n) + await connSrc.readExactly(addr msg[0], n) + var ip: array[4, byte] + for i, e in msg[0..^3]: + ip[i] = e + $(ipv4(ip)) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1]))) + of Socks5AddressType.IPv6.byte: + let n = 16 + 2 # +2 bytes for the port + msg = newSeq[byte](n) # +2 bytes for the port + await connSrc.readExactly(addr msg[0], n) + var ip: array[16, byte] + for i, e in msg[0..^3]: + ip[i] = e + $(ipv6(ip)) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1]))) + of Socks5AddressType.FQDN.byte: + await connSrc.readExactly(addr msg[0], 1) + let n = int(uint8.fromBytes(msg[0..0])) + 2 # +2 bytes for the port + msg = newSeq[byte](n) + await connSrc.readExactly(addr msg[0], n) + string.fromBytes(msg[0..^3]) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1]))) + else: + raise newException(LPError, "Address not supported") + + let tcpIpAddr = self.addrTable[$(address)] + + await connSrc.write(@[05'u8, 00, 00, 01, 00, 00, 00, 00, 00, 00]) + + let connDst = await self.tcpTransport.dial("", MultiAddress.init(tcpIpAddr).tryGet()) + + await bridge(connSrc, connDst) + await allFutures(connSrc.close(), connDst.close()) + + +proc stop*(self: TorServerStub) {.async.} = + await self.tcpTransport.stop() diff --git a/tests/testnative.nim b/tests/testnative.nim index f35971b4b..cc04fab66 100644 --- a/tests/testnative.nim +++ b/tests/testnative.nim @@ -23,6 +23,7 @@ import testmultibase, testrouting_record import testtcptransport, + testtortransport, testnameresolve, testwstransport, testmultistream, diff --git a/tests/testtcptransport.nim b/tests/testtcptransport.nim index 64d5d8f6b..d0666e1d3 100644 --- a/tests/testtcptransport.nim +++ b/tests/testtcptransport.nim @@ -125,7 +125,8 @@ suite "TCP transport": server.close() await server.join() + proc transProvider(): Transport = TcpTransport.new(upgrade = Upgrade()) + commonTransportTest( - "TcpTransport", - proc (): Transport = TcpTransport.new(upgrade = Upgrade()), + transProvider, "/ip4/0.0.0.0/tcp/0") diff --git a/tests/testtortransport.nim b/tests/testtortransport.nim new file mode 100644 index 000000000..de9050e5a --- /dev/null +++ b/tests/testtortransport.nim @@ -0,0 +1,143 @@ +{.used.} + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import tables +import chronos, stew/[byteutils] +import ../libp2p/[stream/connection, + transports/tcptransport, + transports/tortransport, + upgrademngrs/upgrade, + multiaddress, + builders] + +import ./helpers, ./stubs, ./commontransport + +const torServer = initTAddress("127.0.0.1", 9050.Port) +var stub: TorServerStub +var startFut: Future[void] +suite "Tor transport": + setup: + stub = TorServerStub.new() + stub.registerAddr("127.0.0.1:8080", "/ip4/127.0.0.1/tcp/8080") + stub.registerAddr("libp2p.nim:8080", "/ip4/127.0.0.1/tcp/8080") + stub.registerAddr("::1:8080", "/ip6/::1/tcp/8080") + stub.registerAddr("a2mncbqsbullu7thgm4e6zxda2xccmcgzmaq44oayhdtm6rav5vovcad.onion:80", "/ip4/127.0.0.1/tcp/8080") + stub.registerAddr("a2mncbqsbullu7thgm4e6zxda2xccmcgzmaq44oayhdtm6rav5vovcae.onion:81", "/ip4/127.0.0.1/tcp/8081") + startFut = stub.start(torServer) + teardown: + waitFor startFut.cancelAndWait() + waitFor stub.stop() + checkTrackers() + + proc test(lintesAddr: string, dialAddr: string) {.async.} = + let server = TcpTransport.new({ReuseAddr}, Upgrade()) + let ma2 = @[MultiAddress.init(lintesAddr).tryGet()] + await server.start(ma2) + + proc runClient() {.async.} = + let client = TorTransport.new(transportAddress = torServer, upgrade = Upgrade()) + let conn = await client.dial("", MultiAddress.init(dialAddr).tryGet()) + + await conn.write("client") + var resp: array[6, byte] + await conn.readExactly(addr resp, 6) + await conn.close() + + check string.fromBytes(resp) == "server" + await client.stop() + + proc serverAcceptHandler() {.async, gcsafe.} = + let conn = await server.accept() + var resp: array[6, byte] + await conn.readExactly(addr resp, 6) + check string.fromBytes(resp) == "client" + + await conn.write("server") + await conn.close() + await server.stop() + + asyncSpawn serverAcceptHandler() + await runClient() + + asyncTest "test start and dial using ipv4": + await test("/ip4/127.0.0.1/tcp/8080", "/ip4/127.0.0.1/tcp/8080") + + asyncTest "test start and dial using ipv6": + await test("/ip6/::1/tcp/8080", "/ip6/::1/tcp/8080") + + asyncTest "test start and dial using dns": + await test("/ip4/127.0.0.1/tcp/8080", "/dns/libp2p.nim/tcp/8080") + + asyncTest "test start and dial usion onion3 and builder": + const TestCodec = "/test/proto/1.0.0" # custom protocol string identifier + + type + TestProto = ref object of LPProtocol # declare a custom protocol + + proc new(T: typedesc[TestProto]): T = + + # every incoming connections will be in handled in this closure + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + + var resp: array[6, byte] + await conn.readExactly(addr resp, 6) + check string.fromBytes(resp) == "client" + await conn.write("server") + + # We must close the connections ourselves when we're done with it + await conn.close() + + return T(codecs: @[TestCodec], handler: handle) + + let rng = newRng() + + let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/8080/onion3/a2mncbqsbullu7thgm4e6zxda2xccmcgzmaq44oayhdtm6rav5vovcad:80").tryGet() + + let serverSwitch = TorSwitch.new(torServer, rng, @[ma], {ReuseAddr}) + + # setup the custom proto + let testProto = TestProto.new() + + serverSwitch.mount(testProto) + await serverSwitch.start() + + let serverPeerId = serverSwitch.peerInfo.peerId + let serverAddress = serverSwitch.peerInfo.addrs + + proc startClient() {.async.} = + let clientSwitch = TorSwitch.new(torServer = torServer, rng= rng, flags = {ReuseAddr}) + + let conn = await clientSwitch.dial(serverPeerId, serverAddress, TestCodec) + + await conn.write("client") + + var resp: array[6, byte] + await conn.readExactly(addr resp, 6) + check string.fromBytes(resp) == "server" + await conn.close() + await clientSwitch.stop() + + await startClient() + + await serverSwitch.stop() + + test "It's not possible to add another transport in TorSwitch": + when (NimMajor, NimMinor, NimPatch) < (1, 4, 0): + type AssertionDefect = AssertionError + + let torSwitch = TorSwitch.new(torServer = torServer, rng= rng, flags = {ReuseAddr}) + expect(AssertionDefect): + torSwitch.addTransport(TcpTransport.new(upgrade = Upgrade())) + waitFor torSwitch.stop() + + proc transProvider(): Transport = + TorTransport.new(torServer, {ReuseAddr}, Upgrade()) + + commonTransportTest( + transProvider, + "/ip4/127.0.0.1/tcp/8080/onion3/a2mncbqsbullu7thgm4e6zxda2xccmcgzmaq44oayhdtm6rav5vovcad:80", + "/ip4/127.0.0.1/tcp/8081/onion3/a2mncbqsbullu7thgm4e6zxda2xccmcgzmaq44oayhdtm6rav5vovcae:81") diff --git a/tests/testwstransport.nim b/tests/testwstransport.nim index 0e46b009e..c2ac88751 100644 --- a/tests/testwstransport.nim +++ b/tests/testwstransport.nim @@ -55,14 +55,13 @@ suite "WebSocket transport": teardown: checkTrackers() - commonTransportTest( - "WebSocket", - proc (): Transport = WsTransport.new(Upgrade()), - "/ip4/0.0.0.0/tcp/0/ws") + proc wsTraspProvider(): Transport = WsTransport.new(Upgrade()) commonTransportTest( - "WebSocket Secure", - (proc (): Transport {.gcsafe.} = + wsTraspProvider, + "/ip4/0.0.0.0/tcp/0/ws") + + proc wsSecureTranspProvider(): Transport {.gcsafe.} = try: return WsTransport.new( Upgrade(), @@ -70,7 +69,9 @@ suite "WebSocket transport": TLSCertificate.init(SecureCert), {TLSFlags.NoVerifyHost, TLSFlags.NoVerifyServerName}) except Exception: check(false) - ), + + commonTransportTest( + wsSecureTranspProvider, "/ip4/0.0.0.0/tcp/0/wss") asyncTest "Hostname verification": From 6ab6ab48ef2013c8cec39e78b810356d6bf28176 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Sat, 5 Nov 2022 02:04:05 +0100 Subject: [PATCH 22/23] Fix after gcsafe requirements change and other issues on devel (#795) --- libp2p.nimble | 1 + libp2p/protocols/pubsub/pubsub.nim | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libp2p.nimble b/libp2p.nimble index ff4f8bd65..301e7c165 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -108,6 +108,7 @@ task examples_build, "Build the samples": buildSample("tutorial_4_gossipsub", true) buildSample("tutorial_5_discovery", true) # Nico doesn't work in 1.2 + exec "nimble install -y nimpng@#HEAD" # this is to fix broken build on 1.7.3, remove it when nimpng version 0.3.2 or later is released exec "nimble install -y nico" buildSample("tutorial_6_game", false, "--styleCheck:off") diff --git a/libp2p/protocols/pubsub/pubsub.nim b/libp2p/protocols/pubsub/pubsub.nim index a7d7fc210..ab010a366 100644 --- a/libp2p/protocols/pubsub/pubsub.nim +++ b/libp2p/protocols/pubsub/pubsub.nim @@ -130,7 +130,7 @@ type knownTopics*: HashSet[string] -method unsubscribePeer*(p: PubSub, peerId: PeerId) {.base.} = +method unsubscribePeer*(p: PubSub, peerId: PeerId) {.base, gcsafe.} = ## handle peer disconnects ## @@ -377,7 +377,7 @@ method handleConn*(p: PubSub, finally: await conn.closeWithEOF() -method subscribePeer*(p: PubSub, peer: PeerId) {.base.} = +method subscribePeer*(p: PubSub, peer: PeerId) {.base, gcsafe.} = ## subscribe to remote peer to receive/send pubsub ## messages ## @@ -400,7 +400,7 @@ proc updateTopicMetrics(p: PubSub, topic: string) = libp2p_pubsub_topic_handlers.set(others, labelValues = ["other"]) -method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base.} = +method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base, gcsafe.} = # Called when subscribe is called the first time for a topic or unsubscribe # removes the last handler @@ -433,7 +433,7 @@ proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) {.public.} = for t in topics: p.unsubscribe(t.topic, t.handler) -proc unsubscribeAll*(p: PubSub, topic: string) {.public.} = +proc unsubscribeAll*(p: PubSub, topic: string) {.public, gcsafe.} = ## unsubscribe every `handler` from `topic` if topic notin p.topics: debug "unsubscribeAll called for an unknown topic", topic @@ -495,7 +495,7 @@ method initPubSub*(p: PubSub) method addValidator*(p: PubSub, topic: varargs[string], - hook: ValidatorHandler) {.base, public.} = + hook: ValidatorHandler) {.base, public, gcsafe.} = ## Add a validator to a `topic`. Each new message received in this ## will be sent to `hook`. `hook` can return either `Accept`, ## `Ignore` or `Reject` (which can descore the peer) From 23338fceaa6200c2aa6e8d98db381ff72413acda Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Mon, 7 Nov 2022 22:55:22 +0100 Subject: [PATCH 23/23] Explicit dependency to unittest2 and update other packages (#797) --- .pinned | 6 +++--- libp2p.nimble | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pinned b/.pinned index 57b4bb89d..88d897cbc 100644 --- a/.pinned +++ b/.pinned @@ -1,6 +1,6 @@ bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a -chronos;https://github.com/status-im/nim-chronos@#266e2c0ed26b455872bccb3ddbd316815a283659 +chronos;https://github.com/status-im/nim-chronos@#6525f4ce1d1a7eba146e5f1a53f6f105077ae686 dnsclient;https://github.com/ba0f3/dnsclient.nim@#fcd7443634b950eaea574e5eaa00a628ae029823 faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29 httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040 @@ -8,9 +8,9 @@ json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb7 metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178 secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f -serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466 +serialization;https://github.com/status-im/nim-serialization@#60a5bd8ac0461dfadd3069fd9c01a7734f205995 stew;https://github.com/status-im/nim-stew@#23da07c9b59c0ba3d4efa7e4e6e2c4121ae5a156 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34 -unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db +unittest2;https://github.com/status-im/nim-unittest2@#da8398c45cafd5bd7772da1fc96e3924a18d3823 websock;https://github.com/status-im/nim-websock@#acbe30e9ca1e51dcbbfe4c552ee6f16c7eede538 zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8 \ No newline at end of file diff --git a/libp2p.nimble b/libp2p.nimble index 301e7c165..7dd20dafc 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -16,7 +16,8 @@ requires "nim >= 1.2.0", "metrics", "secp256k1", "stew#head", - "websock" + "websock", + "unittest2 >= 0.0.5 & < 0.1.0" proc runTest(filename: string, verify: bool = true, sign: bool = true, moreoptions: string = "") =