From dfbfbe6eb6b641035c106be34b66b707ce27c849 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 5 Sep 2022 14:31:14 +0200 Subject: [PATCH 01/20] allow connection to a peer with unknown PeerId (#756) Co-authored-by: Tanguy --- libp2p/crypto/curve25519.nim | 3 +- libp2p/dial.nim | 7 +++++ libp2p/dialer.nim | 31 +++++++++++-------- libp2p/peerid.nim | 2 +- libp2p/protocols/secure/noise.nim | 45 +++++++++++++--------------- libp2p/protocols/secure/secio.nim | 13 +++++--- libp2p/protocols/secure/secure.nim | 15 ++++++---- libp2p/switch.nim | 7 +++++ libp2p/transports/transport.nim | 5 ++-- libp2p/upgrademngrs/muxedupgrade.nim | 7 +++-- libp2p/upgrademngrs/upgrade.nim | 8 +++-- tests/testnoise.nim | 20 +++++-------- tests/testswitch.nim | 14 +++++++++ 13 files changed, 108 insertions(+), 69 deletions(-) diff --git a/libp2p/crypto/curve25519.nim b/libp2p/crypto/curve25519.nim index d4b476b..98a80d7 100644 --- a/libp2p/crypto/curve25519.nim +++ b/libp2p/crypto/curve25519.nim @@ -31,7 +31,6 @@ const type Curve25519* = object Curve25519Key* = array[Curve25519KeySize, byte] - pcuchar = ptr char Curve25519Error* = enum Curver25519GenError @@ -77,7 +76,7 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) = addr rpoint[0], Curve25519KeySize, EC_curve25519) - + assert size == Curve25519KeySize proc public*(private: Curve25519Key): Curve25519Key = diff --git a/libp2p/dial.nim b/libp2p/dial.nim index bfe1620..bb8d00c 100644 --- a/libp2p/dial.nim +++ b/libp2p/dial.nim @@ -31,6 +31,13 @@ method connect*( doAssert(false, "Not implemented!") +method connect*( + self: Dial, + addrs: seq[MultiAddress]): Future[PeerId] {.async, base.} = + ## Connects to a peer and retrieve its PeerId + + doAssert(false, "Not implemented!") + method dial*( self: Dial, peerId: PeerId, diff --git a/libp2p/dialer.nim b/libp2p/dialer.nim index 532ca36..85c5b63 100644 --- a/libp2p/dialer.nim +++ b/libp2p/dialer.nim @@ -47,7 +47,7 @@ type proc dialAndUpgrade( self: Dialer, - peerId: PeerId, + peerId: Opt[PeerId], addrs: seq[MultiAddress]): Future[Connection] {.async.} = debug "Dialing peer", peerId @@ -74,9 +74,6 @@ proc dialAndUpgrade( libp2p_failed_dials.inc() continue # Try the next address - # make sure to assign the peer to the connection - dialed.peerId = peerId - # also keep track of the connection's bottom unsafe transport direction # required by gossipsub scoring dialed.transportDir = Direction.Out @@ -84,7 +81,7 @@ proc dialAndUpgrade( libp2p_successful_dials.inc() let conn = try: - await transport.upgradeOutgoing(dialed) + await transport.upgradeOutgoing(dialed, peerId) except CatchableError as exc: # If we failed to establish the connection through one transport, # we won't succeeded through another - no use in trying again @@ -101,20 +98,22 @@ proc dialAndUpgrade( proc internalConnect( self: Dialer, - peerId: PeerId, + peerId: Opt[PeerId], addrs: seq[MultiAddress], forceDial: bool): Future[Connection] {.async.} = - if self.localPeerId == peerId: + if Opt.some(self.localPeerId) == peerId: raise newException(CatchableError, "can't dial self!") # Ensure there's only one in-flight attempt per peer - let lock = self.dialLock.mgetOrPut(peerId, newAsyncLock()) + let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock()) try: await lock.acquire() # Check if we have a connection already and try to reuse it - var conn = self.connManager.selectConn(peerId) + var conn = + if peerId.isSome: self.connManager.selectConn(peerId.get()) + else: nil if conn != nil: if conn.atEof or conn.closed: # This connection should already have been removed from the connection @@ -165,7 +164,15 @@ method connect*( if self.connManager.connCount(peerId) > 0: return - discard await self.internalConnect(peerId, addrs, forceDial) + discard await self.internalConnect(Opt.some(peerId), addrs, forceDial) + +method connect*( + self: Dialer, + addrs: seq[MultiAddress], + ): Future[PeerId] {.async.} = + ## Connects to a peer and retrieve its PeerId + + return (await self.internalConnect(Opt.none(PeerId), addrs, false)).peerId proc negotiateStream( self: Dialer, @@ -190,7 +197,7 @@ method tryDial*( trace "Check if it can dial", peerId, addrs try: - let conn = await self.dialAndUpgrade(peerId, addrs) + let conn = await self.dialAndUpgrade(Opt.some(peerId), addrs) if conn.isNil(): raise newException(DialFailedError, "No valid multiaddress") await conn.close() @@ -238,7 +245,7 @@ method dial*( try: trace "Dialing (new)", peerId, protos - conn = await self.internalConnect(peerId, addrs, forceDial) + conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial) trace "Opening stream", conn stream = await self.connManager.getStream(conn) diff --git a/libp2p/peerid.nim b/libp2p/peerid.nim index dc7a4ae..401fee9 100644 --- a/libp2p/peerid.nim +++ b/libp2p/peerid.nim @@ -43,7 +43,7 @@ func shortLog*(pid: PeerId): string = var spid = $pid if len(spid) > 10: spid[3] = '*' - + when (NimMajor, NimMinor) > (1, 4): spid.delete(4 .. spid.high - 6) else: diff --git a/libp2p/protocols/secure/noise.nim b/libp2p/protocols/secure/noise.nim index 6e613ea..66a8dcd 100644 --- a/libp2p/protocols/secure/noise.nim +++ b/libp2p/protocols/secure/noise.nim @@ -38,7 +38,7 @@ const # https://godoc.org/github.com/libp2p/go-libp2p-noise#pkg-constants NoiseCodec* = "/noise" - PayloadString = "noise-libp2p-static-key:" + PayloadString = toBytes("noise-libp2p-static-key:") ProtocolXXName = "Noise_XX_25519_ChaChaPoly_SHA256" @@ -339,7 +339,6 @@ proc handshakeXXOutbound( hs = HandshakeState.init() try: - hs.ss.mixHash(p.commonPrologue) hs.s = p.noiseKeys @@ -445,7 +444,6 @@ method readMessage*(sconn: NoiseConnection): Future[seq[byte]] {.async.} = dumpMessage(sconn, FlowDirection.Incoming, []) trace "Received 0-length message", sconn - proc encryptFrame( sconn: NoiseConnection, cipherFrame: var openArray[byte], @@ -506,7 +504,7 @@ method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] = # sequencing issues sconn.stream.write(cipherFrames) -method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureConn] {.async.} = +method handshake*(p: Noise, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} = trace "Starting Noise handshake", conn, initiator let timeout = conn.timeout @@ -515,7 +513,7 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon # https://github.com/libp2p/specs/tree/master/noise#libp2p-data-in-handshake-messages let signedPayload = p.localPrivateKey.sign( - PayloadString.toBytes & p.noiseKeys.publicKey.getBytes).tryGet() + PayloadString & p.noiseKeys.publicKey.getBytes).tryGet() var libp2pProof = initProtoBuffer() @@ -538,11 +536,9 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon remoteSig: Signature remoteSigBytes: seq[byte] - let r1 = remoteProof.getField(1, remotePubKeyBytes) - let r2 = remoteProof.getField(2, remoteSigBytes) - if r1.isErr() or not(r1.get()): + if not remoteProof.getField(1, remotePubKeyBytes).valueOr(false): raise newException(NoiseHandshakeError, "Failed to deserialize remote public key bytes. (initiator: " & $initiator & ")") - if r2.isErr() or not(r2.get()): + if not remoteProof.getField(2, remoteSigBytes).valueOr(false): raise newException(NoiseHandshakeError, "Failed to deserialize remote signature bytes. (initiator: " & $initiator & ")") if not remotePubKey.init(remotePubKeyBytes): @@ -550,33 +546,34 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon if not remoteSig.init(remoteSigBytes): raise newException(NoiseHandshakeError, "Failed to decode remote signature. (initiator: " & $initiator & ")") - let verifyPayload = PayloadString.toBytes & handshakeRes.rs.getBytes + let verifyPayload = PayloadString & handshakeRes.rs.getBytes if not remoteSig.verify(verifyPayload, remotePubKey): raise newException(NoiseHandshakeError, "Noise handshake signature verify failed.") else: trace "Remote signature verified", conn - if initiator: - let pid = PeerId.init(remotePubKey) - if not conn.peerId.validate(): - raise newException(NoiseHandshakeError, "Failed to validate peerId.") - if pid.isErr or pid.get() != conn.peerId: + let pid = PeerId.init(remotePubKey).valueOr: + raise newException(NoiseHandshakeError, "Invalid remote peer id: " & $error) + + trace "Remote peer id", pid = $pid + + if peerId.isSome(): + let targetPid = peerId.get() + if not targetPid.validate(): + raise newException(NoiseHandshakeError, "Failed to validate expected peerId.") + + if pid != targetPid: var failedKey: PublicKey - discard extractPublicKey(conn.peerId, failedKey) - debug "Noise handshake, peer infos don't match!", + discard extractPublicKey(targetPid, failedKey) + debug "Noise handshake, peer id doesn't match!", initiator, dealt_peer = conn, dealt_key = $failedKey, received_peer = $pid, received_key = $remotePubKey - raise newException(NoiseHandshakeError, "Noise handshake, peer infos don't match! " & $pid & " != " & $conn.peerId) - else: - let pid = PeerId.init(remotePubKey) - if pid.isErr: - raise newException(NoiseHandshakeError, "Invalid remote peer id") - conn.peerId = pid.get() + raise newException(NoiseHandshakeError, "Noise handshake, peer id don't match! " & $pid & " != " & $targetPid) + conn.peerId = pid var tmp = NoiseConnection.new(conn, conn.peerId, conn.observedAddr) - if initiator: tmp.readCs = handshakeRes.cs2 tmp.writeCs = handshakeRes.cs1 diff --git a/libp2p/protocols/secure/secio.nim b/libp2p/protocols/secure/secio.nim index 46a05c7..1ebea90 100644 --- a/libp2p/protocols/secure/secio.nim +++ b/libp2p/protocols/secure/secio.nim @@ -291,7 +291,7 @@ proc transactMessage(conn: Connection, await conn.write(msg) return await conn.readRawMessage() -method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[SecureConn] {.async.} = +method handshake*(s: Secio, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} = var localNonce: array[SecioNonceSize, byte] remoteNonce: seq[byte] @@ -342,9 +342,14 @@ method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[S remotePeerId = PeerId.init(remotePubkey).tryGet() - # TODO: PeerId check against supplied PeerId - if not initiator: - conn.peerId = remotePeerId + if peerId.isSome(): + let targetPid = peerId.get() + if not targetPid.validate(): + raise newException(SecioError, "Failed to validate expected peerId.") + + if remotePeerId != targetPid: + raise newException(SecioError, "Peer ids don't match!") + conn.peerId = remotePeerId let order = getOrder(remoteBytesPubkey, localNonce, localBytesPubkey, remoteNonce).tryGet() trace "Remote proposal", schemes = remoteExchanges, ciphers = remoteCiphers, diff --git a/libp2p/protocols/secure/secure.nim b/libp2p/protocols/secure/secure.nim index f841a5d..0bdf852 100644 --- a/libp2p/protocols/secure/secure.nim +++ b/libp2p/protocols/secure/secure.nim @@ -79,13 +79,15 @@ method getWrapped*(s: SecureConn): Connection = s.stream method handshake*(s: Secure, conn: Connection, - initiator: bool): Future[SecureConn] {.async, base.} = + initiator: bool, + peerId: Opt[PeerId]): Future[SecureConn] {.async, base.} = doAssert(false, "Not implemented!") proc handleConn(s: Secure, conn: Connection, - initiator: bool): Future[Connection] {.async.} = - var sconn = await s.handshake(conn, initiator) + initiator: bool, + peerId: Opt[PeerId]): Future[Connection] {.async.} = + var sconn = await s.handshake(conn, initiator, peerId) # mark connection bottom level transport direction # this is the safest place to do this # we require this information in for example gossipsub @@ -121,7 +123,7 @@ method init*(s: Secure) = try: # We don't need the result but we # definitely need to await the handshake - discard await s.handleConn(conn, false) + discard await s.handleConn(conn, false, Opt.none(PeerId)) trace "connection secured", conn except CancelledError as exc: warn "securing connection canceled", conn @@ -135,9 +137,10 @@ method init*(s: Secure) = method secure*(s: Secure, conn: Connection, - initiator: bool): + initiator: bool, + peerId: Opt[PeerId]): Future[Connection] {.base.} = - s.handleConn(conn, initiator) + s.handleConn(conn, initiator, peerId) method readOnce*(s: SecureConn, pbytes: pointer, diff --git a/libp2p/switch.nim b/libp2p/switch.nim index f6f2925..3bbdaa6 100644 --- a/libp2p/switch.nim +++ b/libp2p/switch.nim @@ -128,6 +128,13 @@ method connect*( s.dialer.connect(peerId, addrs, forceDial) +method connect*( + s: Switch, + addrs: seq[MultiAddress]): Future[PeerId] = + ## Connects to a peer and retrieve its PeerId + + s.dialer.connect(addrs) + method dial*( s: Switch, peerId: PeerId, diff --git a/libp2p/transports/transport.nim b/libp2p/transports/transport.nim index 951f8bf..12d1a08 100644 --- a/libp2p/transports/transport.nim +++ b/libp2p/transports/transport.nim @@ -87,12 +87,13 @@ method upgradeIncoming*( method upgradeOutgoing*( self: Transport, - conn: Connection): Future[Connection] {.base, gcsafe.} = + conn: Connection, + peerId: Opt[PeerId]): Future[Connection] {.base, gcsafe.} = ## base upgrade method that the transport uses to perform ## transport specific upgrades ## - self.upgrader.upgradeOutgoing(conn) + self.upgrader.upgradeOutgoing(conn, peerId) method handles*( self: Transport, diff --git a/libp2p/upgrademngrs/muxedupgrade.nim b/libp2p/upgrademngrs/muxedupgrade.nim index f60d0c1..030508c 100644 --- a/libp2p/upgrademngrs/muxedupgrade.nim +++ b/libp2p/upgrademngrs/muxedupgrade.nim @@ -88,10 +88,11 @@ proc mux*( method upgradeOutgoing*( self: MuxedUpgrade, - conn: Connection): Future[Connection] {.async, gcsafe.} = + conn: Connection, + peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} = trace "Upgrading outgoing connection", conn - let sconn = await self.secure(conn) # secure the connection + let sconn = await self.secure(conn, peerId) # secure the connection if isNil(sconn): raise newException(UpgradeFailedError, "unable to secure connection, stopping upgrade") @@ -129,7 +130,7 @@ method upgradeIncoming*( var cconn = conn try: - var sconn = await secure.secure(cconn, false) + var sconn = await secure.secure(cconn, false, Opt.none(PeerId)) if isNil(sconn): return diff --git a/libp2p/upgrademngrs/upgrade.nim b/libp2p/upgrademngrs/upgrade.nim index 781a074..c5733e6 100644 --- a/libp2p/upgrademngrs/upgrade.nim +++ b/libp2p/upgrademngrs/upgrade.nim @@ -47,12 +47,14 @@ method upgradeIncoming*( method upgradeOutgoing*( self: Upgrade, - conn: Connection): Future[Connection] {.base.} = + conn: Connection, + peerId: Opt[PeerId]): Future[Connection] {.base.} = doAssert(false, "Not implemented!") proc secure*( self: Upgrade, - conn: Connection): Future[Connection] {.async, gcsafe.} = + conn: Connection, + peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} = if self.secureManagers.len <= 0: raise newException(UpgradeFailedError, "No secure managers registered!") @@ -67,7 +69,7 @@ proc secure*( # let's avoid duplicating checks but detect if it fails to do it properly doAssert(secureProtocol.len > 0) - return await secureProtocol[0].secure(conn, true) + return await secureProtocol[0].secure(conn, true, peerId) proc identify*( self: Upgrade, diff --git a/tests/testnoise.nim b/tests/testnoise.nim index eabe56b..b0e785b 100644 --- a/tests/testnoise.nim +++ b/tests/testnoise.nim @@ -104,7 +104,7 @@ suite "Noise": proc acceptHandler() {.async.} = let conn = await transport1.accept() - let sconn = await serverNoise.secure(conn, false) + let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId)) try: await sconn.write("Hello!") finally: @@ -119,8 +119,7 @@ suite "Noise": clientNoise = Noise.new(rng, clientPrivKey, outgoing = true) conn = await transport2.dial(transport1.addrs[0]) - conn.peerId = serverInfo.peerId - let sconn = await clientNoise.secure(conn, true) + let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId)) var msg = newSeq[byte](6) await sconn.readExactly(addr msg[0], 6) @@ -149,7 +148,7 @@ suite "Noise": var conn: Connection try: conn = await transport1.accept() - discard await serverNoise.secure(conn, false) + discard await serverNoise.secure(conn, false, Opt.none(PeerId)) except CatchableError: discard finally: @@ -162,11 +161,10 @@ suite "Noise": clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs) clientNoise = Noise.new(rng, clientPrivKey, outgoing = true, commonPrologue = @[1'u8, 2'u8, 3'u8]) conn = await transport2.dial(transport1.addrs[0]) - conn.peerId = serverInfo.peerId var sconn: Connection = nil expect(NoiseDecryptTagError): - sconn = await clientNoise.secure(conn, true) + sconn = await clientNoise.secure(conn, true, Opt.some(conn.peerId)) await conn.close() await handlerWait @@ -186,7 +184,7 @@ suite "Noise": proc acceptHandler() {.async, gcsafe.} = let conn = await transport1.accept() - let sconn = await serverNoise.secure(conn, false) + let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId)) defer: await sconn.close() await conn.close() @@ -202,8 +200,7 @@ suite "Noise": clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs) clientNoise = Noise.new(rng, clientPrivKey, outgoing = true) conn = await transport2.dial(transport1.addrs[0]) - conn.peerId = serverInfo.peerId - let sconn = await clientNoise.secure(conn, true) + let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId)) await sconn.write("Hello!") await acceptFut @@ -230,7 +227,7 @@ suite "Noise": proc acceptHandler() {.async, gcsafe.} = let conn = await transport1.accept() - let sconn = await serverNoise.secure(conn, false) + let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId)) defer: await sconn.close() let msg = await sconn.readLp(1024*1024) @@ -244,8 +241,7 @@ suite "Noise": clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs) clientNoise = Noise.new(rng, clientPrivKey, outgoing = true) conn = await transport2.dial(transport1.addrs[0]) - conn.peerId = serverInfo.peerId - let sconn = await clientNoise.secure(conn, true) + let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId)) await sconn.writeLp(hugePayload) await readTask diff --git a/tests/testswitch.nim b/tests/testswitch.nim index daaa0f0..3d47b1a 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -201,6 +201,20 @@ suite "Switch": check not switch1.isConnected(switch2.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) + asyncTest "e2e connect to peer with unkown PeerId": + let switch1 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) + let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) + await switch1.start() + await switch2.start() + + check: (await switch2.connect(switch1.peerInfo.addrs)) == switch1.peerInfo.peerId + await switch2.disconnect(switch1.peerInfo.peerId) + + await allFuturesThrowing( + switch1.stop(), + switch2.stop() + ) + asyncTest "e2e should not leak on peer disconnect": let switch1 = newStandardSwitch() let switch2 = newStandardSwitch() From abbeaab684c500f4c6ff5881797bb8f184b41ccc Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 6 Sep 2022 13:20:42 +0200 Subject: [PATCH 02/20] Keep connection alive when peer doesn't support pubsub (#754) --- libp2p/protocols/pubsub/pubsub.nim | 10 +--------- libp2p/protocols/pubsub/pubsubpeer.nim | 6 ------ tests/pubsub/testgossipinternal.nim | 5 +---- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/libp2p/protocols/pubsub/pubsub.nim b/libp2p/protocols/pubsub/pubsub.nim index 823d792..606744a 100644 --- a/libp2p/protocols/pubsub/pubsub.nim +++ b/libp2p/protocols/pubsub/pubsub.nim @@ -292,19 +292,11 @@ proc getOrCreatePeer*( proc getConn(): Future[Connection] {.async.} = return await p.switch.dial(peerId, protos) - proc dropConn(peer: PubSubPeer) = - proc dropConnAsync(peer: PubSubPeer) {.async.} = - try: - await p.switch.disconnect(peer.peerId) - except CatchableError as exc: # never cancelled - trace "Failed to close connection", peer, error = exc.name, msg = exc.msg - asyncSpawn dropConnAsync(peer) - proc onEvent(peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe.} = p.onPubSubPeerEvent(peer, event) # create new pubsub peer - let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, onEvent, protos[0], p.maxMessageSize) + let pubSubPeer = PubSubPeer.new(peerId, getConn, onEvent, protos[0], p.maxMessageSize) debug "created new pubsub peer", peerId p.peers[peerId] = pubSubPeer diff --git a/libp2p/protocols/pubsub/pubsubpeer.nim b/libp2p/protocols/pubsub/pubsubpeer.nim index 89f5733..8af49de 100644 --- a/libp2p/protocols/pubsub/pubsubpeer.nim +++ b/libp2p/protocols/pubsub/pubsubpeer.nim @@ -51,7 +51,6 @@ type PubSubPeer* = ref object of RootObj getConn*: GetConn # callback to establish a new send connection - dropConn*: DropConn # Function pointer to use to drop connections onEvent*: OnEvent # Connectivity updates for peer codec*: string # the protocol that this peer joined from sendConn*: Connection # cached send connection @@ -206,9 +205,6 @@ proc connectImpl(p: PubSubPeer) {.async.} = await connectOnce(p) except CatchableError as exc: # never cancelled debug "Could not establish send connection", msg = exc.msg - finally: - # drop the connection, else we end up with ghost peers - if p.dropConn != nil: p.dropConn(p) proc connect*(p: PubSubPeer) = asyncSpawn connectImpl(p) @@ -286,14 +282,12 @@ proc new*( T: typedesc[PubSubPeer], peerId: PeerId, getConn: GetConn, - dropConn: DropConn, onEvent: OnEvent, codec: string, maxMessageSize: int): T = T( getConn: getConn, - dropConn: dropConn, onEvent: onEvent, codec: codec, peerId: peerId, diff --git a/tests/pubsub/testgossipinternal.nim b/tests/pubsub/testgossipinternal.nim index 84f75e4..91ad4c0 100644 --- a/tests/pubsub/testgossipinternal.nim +++ b/tests/pubsub/testgossipinternal.nim @@ -22,10 +22,7 @@ proc getPubSubPeer(p: TestGossipSub, peerId: PeerId): PubSubPeer = proc getConn(): Future[Connection] = p.switch.dial(peerId, GossipSubCodec) - proc dropConn(peer: PubSubPeer) = - discard # we don't care about it here yet - - let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, nil, GossipSubCodec, 1024 * 1024) + let pubSubPeer = PubSubPeer.new(peerId, getConn, nil, GossipSubCodec, 1024 * 1024) debug "created new pubsub peer", peerId p.peers[peerId] = pubSubPeer From d8a9e93ff7fc28c2c636670f905c1c4c990acc98 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Thu, 8 Sep 2022 17:10:11 +0200 Subject: [PATCH 03/20] Add onion3 multiaddr support (#764) --- libp2p/multiaddress.nim | 43 ++++++++++++++++++++++++++++++++++++++ libp2p/multicodec.nim | 1 + tests/testmultiaddress.nim | 26 +++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index 9fd636c..054fb00 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -222,6 +222,40 @@ proc onionVB(vb: var VBuffer): bool = if vb.readArray(buf) == 12: result = true +proc onion3StB(s: string, vb: var VBuffer): bool = + try: + var parts = s.split(':') + if len(parts) != 2: + return false + if len(parts[0]) != 56: + return false + var address = Base32Lower.decode(parts[0].toLowerAscii()) + var nport = parseInt(parts[1]) + if (nport > 0 and nport < 65536) and len(address) == 35: + address.setLen(37) + address[35] = cast[byte]((nport shr 8) and 0xFF) + address[36] = cast[byte](nport and 0xFF) + vb.writeArray(address) + result = true + except: + discard + +proc onion3BtS(vb: var VBuffer, s: var string): bool = + ## ONION address bufferToString() implementation. + var buf: array[37, byte] + if vb.readArray(buf) == 37: + var nport = (cast[uint16](buf[35]) shl 8) or cast[uint16](buf[36]) + s = Base32Lower.encode(buf.toOpenArray(0, 34)) + s.add(":") + s.add($nport) + result = true + +proc onion3VB(vb: var VBuffer): bool = + ## ONION address validateBuffer() implementation. + var buf: array[37, byte] + if vb.readArray(buf) == 37: + result = true + proc unixStB(s: string, vb: var VBuffer): bool = ## Unix socket name stringToBuffer() implementation. if len(s) > 0: @@ -310,6 +344,11 @@ const bufferToString: onionBtS, validateBuffer: onionVB ) + TranscoderOnion3* = Transcoder( + stringToBuffer: onion3StB, + bufferToString: onion3BtS, + validateBuffer: onion3VB + ) TranscoderDNS* = Transcoder( stringToBuffer: dnsStB, bufferToString: dnsBtS, @@ -363,6 +402,10 @@ const mcodec: multiCodec("onion"), kind: Fixed, size: 10, coder: TranscoderOnion ), + MAProtocol( + mcodec: multiCodec("onion3"), kind: Fixed, size: 37, + coder: TranscoderOnion3 + ), MAProtocol( mcodec: multiCodec("ws"), kind: Marker, size: 0 ), diff --git a/libp2p/multicodec.nim b/libp2p/multicodec.nim index 0cfc4c5..a1f7106 100644 --- a/libp2p/multicodec.nim +++ b/libp2p/multicodec.nim @@ -203,6 +203,7 @@ const MultiCodecList = [ ("p2p-webrtc-star", 0x0113), # not in multicodec list ("p2p-webrtc-direct", 0x0114), # not in multicodec list ("onion", 0x01BC), + ("onion3", 0x01BD), ("p2p-circuit", 0x0122), ("libp2p-peer-record", 0x0301), ("dns", 0x35), diff --git a/tests/testmultiaddress.nim b/tests/testmultiaddress.nim index 4ce91f7..2c05e21 100644 --- a/tests/testmultiaddress.nim +++ b/tests/testmultiaddress.nim @@ -22,6 +22,8 @@ const "/ip6zone/x/ip6/fe80::1/udp/1234/quic", "/onion/timaq4ygg2iegci7:1234", "/onion/timaq4ygg2iegci7:80/http", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:1234", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80/http", "/udp/0", "/tcp/0", "/sctp/0", @@ -79,6 +81,12 @@ const "/onion/timaq4ygg2iegci7:-1", "/onion/timaq4ygg2iegci7", "/onion/timaq4ygg2iegci@:666", + "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666", "/udp/1234/sctp", "/udp/1234/udt/1234", "/udp/1234/utp/1234", @@ -170,6 +178,12 @@ const "/onion/timaq4ygg2iegci7:-1", "/onion/timaq4ygg2iegci7", "/onion/timaq4ygg2iegci@:666", + "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666", "/udp/1234/sctp", "/udp/1234/udt/1234", "/udp/1234/utp/1234", @@ -376,3 +390,15 @@ suite "MultiAddress test suite": $ma[1..2].get() == "/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" $ma[^3..^1].get() == "/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio" ma[5..7].isErr() + + test "[](MultiCodec) test": + let onionMAStr = "/onion3/torchdeedp3i2jigzjdmfpn5ttjhthh5wbmda2rr3jvqjg5p77c54dqd:80" + let ma = MultiAddress.init(onionMAStr).get() + check $(ma[multiCodec("onion3")].tryGet()) == onionMAStr + + let onionMAWithTcpStr = "/onion3/torchdeedp3i2jigzjdmfpn5ttjhthh5wbmda2rr3jvqjg5p77c54dqd:80/tcp/80" + let maWithTcp = MultiAddress.init(onionMAWithTcpStr).get() + check $(maWithTcp[multiCodec("onion3")].tryGet()) == onionMAStr + + + From ef594e1e02f31700af530712ba37c3c59ea7ded9 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 12 Sep 2022 17:09:10 +0200 Subject: [PATCH 04/20] Only log multiple missed heartbeats as info (#763) --- libp2p/utils/heartbeat.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libp2p/utils/heartbeat.nim b/libp2p/utils/heartbeat.nim index 6756b16..f0e7a58 100644 --- a/libp2p/utils/heartbeat.nim +++ b/libp2p/utils/heartbeat.nim @@ -25,6 +25,14 @@ template heartbeat*(name: string, interval: Duration, body: untyped): untyped = nextHeartbeat += interval let now = Moment.now() if nextHeartbeat < now: - info "Missed heartbeat", heartbeat = name, delay = now - nextHeartbeat - nextHeartbeat = now + interval + let + delay = now - nextHeartbeat + itv = interval + if delay > itv: + info "Missed multiple heartbeats", heartbeat = name, + delay = delay, hinterval = itv + else: + debug "Missed heartbeat", heartbeat = name, + delay = delay, hinterval = itv + nextHeartbeat = now + itv await sleepAsync(nextHeartbeat - now) From 4d8b50d24ce788d08e0602a51b262bf777332e22 Mon Sep 17 00:00:00 2001 From: lchenut Date: Wed, 14 Sep 2022 10:58:41 +0200 Subject: [PATCH 05/20] Specify EOF error (#759) --- libp2p/muxers/mplex/lpchannel.nim | 21 ++++++++-- libp2p/muxers/mplex/mplex.nim | 1 + libp2p/muxers/yamux/yamux.nim | 65 +++++++++++++++++++------------ libp2p/stream/bufferstream.nim | 2 +- libp2p/stream/lpstream.nim | 38 +++++++++++++++--- tests/testmplex.nim | 10 ++--- tests/testyamux.nim | 33 ++++++++++++++++ 7 files changed, 132 insertions(+), 38 deletions(-) diff --git a/libp2p/muxers/mplex/lpchannel.nim b/libp2p/muxers/mplex/lpchannel.nim index 54d0da5..5f6faa8 100644 --- a/libp2p/muxers/mplex/lpchannel.nim +++ b/libp2p/muxers/mplex/lpchannel.nim @@ -58,6 +58,8 @@ type initiator*: bool # initiated remotely or locally flag isOpen*: bool # has channel been opened closedLocal*: bool # has channel been closed locally + remoteReset*: bool # has channel been remotely reset + localReset*: bool # has channel been reset locally msgCode*: MessageType # cached in/out message code closeCode*: MessageType # cached in/out close code resetCode*: MessageType # cached in/out reset code @@ -103,6 +105,7 @@ proc reset*(s: LPChannel) {.async, gcsafe.} = s.isClosed = true s.closedLocal = true + s.localReset = not s.remoteReset trace "Resetting channel", s, len = s.len @@ -168,6 +171,14 @@ method readOnce*(s: LPChannel, ## channels are blocked - in particular, this means that reading from one ## channel must not be done from within a callback / read handler of another ## or the reads will lock each other. + if s.remoteReset: + raise newLPStreamResetError() + if s.localReset: + raise newLPStreamClosedError() + if s.atEof(): + raise newLPStreamRemoteClosedError() + if s.conn.closed: + raise newLPStreamConnDownError() try: let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes) when defined(libp2p_network_protocols_metrics): @@ -184,13 +195,17 @@ method readOnce*(s: LPChannel, # data has been lost in s.readBuf and there's no way to gracefully recover / # use the channel any more await s.reset() - raise exc + raise newLPStreamConnDownError(exc) proc prepareWrite(s: LPChannel, msg: seq[byte]): Future[void] {.async.} = # prepareWrite is the slow path of writing a message - see conditions in # write - if s.closedLocal or s.conn.closed: + if s.remoteReset: + raise newLPStreamResetError() + if s.closedLocal: raise newLPStreamClosedError() + if s.conn.closed: + raise newLPStreamConnDownError() if msg.len == 0: return @@ -235,7 +250,7 @@ proc completeWrite( trace "exception in lpchannel write handler", s, msg = exc.msg await s.reset() await s.conn.close() - raise exc + raise newLPStreamConnDownError(exc) finally: s.writes -= 1 diff --git a/libp2p/muxers/mplex/mplex.nim b/libp2p/muxers/mplex/mplex.nim index 9083812..fc0294c 100644 --- a/libp2p/muxers/mplex/mplex.nim +++ b/libp2p/muxers/mplex/mplex.nim @@ -183,6 +183,7 @@ method handle*(m: Mplex) {.async, gcsafe.} = of MessageType.CloseIn, MessageType.CloseOut: await channel.pushEof() of MessageType.ResetIn, MessageType.ResetOut: + channel.remoteReset = true await channel.reset() except CancelledError: debug "Unexpected cancellation in mplex handler", m diff --git a/libp2p/muxers/yamux/yamux.nim b/libp2p/muxers/yamux/yamux.nim index 83deb10..a527379 100644 --- a/libp2p/muxers/yamux/yamux.nim +++ b/libp2p/muxers/yamux/yamux.nim @@ -153,6 +153,7 @@ type sendQueue: seq[ToSend] recvQueue: seq[byte] isReset: bool + remoteReset: bool closedRemotely: Future[void] closedLocally: bool receivedData: AsyncEvent @@ -194,23 +195,25 @@ method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} = await channel.actuallyClose() proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} = - if not channel.isReset: - trace "Reset channel" - channel.isReset = true - for (d, s, fut) in channel.sendQueue: - fut.fail(newLPStreamEOFError()) - channel.sendQueue = @[] - channel.recvQueue = @[] - channel.sendWindow = 0 - if not channel.closedLocally: - if isLocal: - try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst})) - except LPStreamEOFError as exc: discard - except LPStreamClosedError as exc: discard - await channel.close() - if not channel.closedRemotely.done(): - await channel.remoteClosed() - channel.receivedData.fire() + if channel.isReset: + return + trace "Reset channel" + channel.isReset = true + channel.remoteReset = not isLocal + for (d, s, fut) in channel.sendQueue: + fut.fail(newLPStreamEOFError()) + channel.sendQueue = @[] + channel.recvQueue = @[] + channel.sendWindow = 0 + if not channel.closedLocally: + if isLocal: + try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst})) + except LPStreamEOFError as exc: discard + except LPStreamClosedError as exc: discard + await channel.close() + if not channel.closedRemotely.done(): + await channel.remoteClosed() + channel.receivedData.fire() if not isLocal: # If we reset locally, we want to flush up to a maximum of recvWindow # bytes. We use the recvWindow in the proc cleanupChann. @@ -235,7 +238,15 @@ method readOnce*( nbytes: int): Future[int] {.async.} = - if channel.returnedEof: raise newLPStreamEOFError() + if channel.isReset: + raise if channel.remoteReset: + newLPStreamResetError() + elif channel.closedLocally: + newLPStreamClosedError() + else: + newLPStreamConnDownError() + if channel.returnedEof: + raise newLPStreamRemoteClosedError() if channel.recvQueue.len == 0: channel.receivedData.clear() await channel.closedRemotely or channel.receivedData.wait() @@ -313,8 +324,9 @@ proc trySend(channel: YamuxChannel) {.async.} = channel.sendWindow.dec(toSend) try: await channel.conn.write(sendBuffer) except CatchableError as exc: + let connDown = newLPStreamConnDownError(exc) for fut in futures.items(): - fut.fail(exc) + fut.fail(connDown) await channel.reset() break for fut in futures.items(): @@ -323,8 +335,11 @@ proc trySend(channel: YamuxChannel) {.async.} = method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] = result = newFuture[void]("Yamux Send") + if channel.remoteReset: + result.fail(newLPStreamResetError()) + return result if channel.closedLocally or channel.isReset: - result.fail(newLPStreamEOFError()) + result.fail(newLPStreamClosedError()) return result if msg.len == 0: result.complete() @@ -396,8 +411,9 @@ method close*(m: Yamux) {.async.} = m.isClosed = true trace "Closing yamux" - for channel in m.channels.values: - await channel.reset() + let channels = toSeq(m.channels.values()) + for channel in channels: + await channel.reset(true) await m.connection.write(YamuxHeader.goAway(NormalTermination)) await m.connection.close() trace "Closed yamux" @@ -453,8 +469,9 @@ method handle*(m: Yamux) {.async, gcsafe.} = m.flushed[header.streamId].dec(int(header.length)) if m.flushed[header.streamId] < 0: raise newException(YamuxError, "Peer exhausted the recvWindow after reset") - var buffer = newSeqUninitialized[byte](header.length) - await m.connection.readExactly(addr buffer[0], int(header.length)) + if header.length > 0: + var buffer = newSeqUninitialized[byte](header.length) + await m.connection.readExactly(addr buffer[0], int(header.length)) continue let channel = m.channels[header.streamId] diff --git a/libp2p/stream/bufferstream.nim b/libp2p/stream/bufferstream.nim index 6eb83ed..68cf862 100644 --- a/libp2p/stream/bufferstream.nim +++ b/libp2p/stream/bufferstream.nim @@ -79,7 +79,7 @@ method pushData*(s: BufferStream, data: seq[byte]) {.base, async.} = &"Only one concurrent push allowed for stream {s.shortLog()}") if s.isClosed or s.pushedEof: - raise newLPStreamEOFError() + raise newLPStreamClosedError() if data.len == 0: return # Don't push 0-length buffers, these signal EOF diff --git a/libp2p/stream/lpstream.nim b/libp2p/stream/lpstream.nim index 6857da3..fb9401a 100644 --- a/libp2p/stream/lpstream.nim +++ b/libp2p/stream/lpstream.nim @@ -59,7 +59,18 @@ type LPStreamWriteError* = object of LPStreamError par*: ref CatchableError LPStreamEOFError* = object of LPStreamError - LPStreamClosedError* = object of LPStreamError + +# X | Read | Write +# Local close | Works | LPStreamClosedError +# Remote close | LPStreamRemoteClosedError | Works +# Local reset | LPStreamClosedError | LPStreamClosedError +# Remote reset | LPStreamResetError | LPStreamResetError +# Connection down | LPStreamConnDown | LPStreamConnDownError + + LPStreamResetError* = object of LPStreamEOFError + LPStreamClosedError* = object of LPStreamEOFError + LPStreamRemoteClosedError* = object of LPStreamEOFError + LPStreamConnDownError* = object of LPStreamEOFError InvalidVarintError* = object of LPStreamError MaxSizeError* = object of LPStreamError @@ -119,9 +130,22 @@ proc newLPStreamIncorrectDefect*(m: string): ref LPStreamIncorrectDefect = proc newLPStreamEOFError*(): ref LPStreamEOFError = result = newException(LPStreamEOFError, "Stream EOF!") +proc newLPStreamResetError*(): ref LPStreamResetError = + result = newException(LPStreamResetError, "Stream Reset!") + proc newLPStreamClosedError*(): ref LPStreamClosedError = result = newException(LPStreamClosedError, "Stream Closed!") +proc newLPStreamRemoteClosedError*(): ref LPStreamRemoteClosedError = + result = newException(LPStreamRemoteClosedError, "Stream Remotely Closed!") + +proc newLPStreamConnDownError*( + parentException: ref Exception = nil): ref LPStreamConnDownError = + result = newException( + LPStreamConnDownError, + "Stream Underlying Connection Closed!", + parentException) + func shortLog*(s: LPStream): auto = if s.isNil: "LPStream(nil)" else: $s.oid @@ -165,6 +189,8 @@ proc readExactly*(s: LPStream, ## Waits for `nbytes` to be available, then read ## them and return them if s.atEof: + var ch: char + discard await s.readOnce(addr ch, 1) raise newLPStreamEOFError() if nbytes == 0: @@ -183,6 +209,10 @@ proc readExactly*(s: LPStream, if read == 0: doAssert s.atEof() trace "couldn't read all bytes, stream EOF", s, nbytes, read + # Re-readOnce to raise a more specific error than EOF + # Raise EOF if it doesn't raise anything(shouldn't happen) + discard await s.readOnce(addr pbuffer[read], nbytes - read) + warn "Read twice while at EOF" raise newLPStreamEOFError() if read < nbytes: @@ -200,8 +230,7 @@ proc readLine*(s: LPStream, while true: var ch: char - if (await readOnce(s, addr ch, 1)) == 0: - raise newLPStreamEOFError() + await readExactly(s, addr ch, 1) if sep[state] == ch: inc(state) @@ -224,8 +253,7 @@ proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe, public.} = buffer: array[10, byte] for i in 0.. Date: Wed, 14 Sep 2022 14:05:43 +0200 Subject: [PATCH 06/20] Add codex & waku to autobump (#768) --- .github/workflows/bumper.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bumper.yml b/.github/workflows/bumper.yml index d16c38f..aee8201 100644 --- a/.github/workflows/bumper.yml +++ b/.github/workflows/bumper.yml @@ -7,14 +7,21 @@ on: workflow_dispatch: jobs: - bumpNimbus: + bumpProjects: runs-on: ubuntu-latest + strategy: + matrix: + target: [ + { repo: status-im/nimbus-eth2, branch: unstable }, + { repo: status-im/nwaku, branch: master }, + { repo: status-im/nim-codex, branch: main } + ] steps: - - name: Clone NBC + - name: Clone repo uses: actions/checkout@v2 with: - repository: status-im/nimbus-eth2 - ref: unstable + repository: ${{ matrix.target.repo }} + ref: ${{ matrix.target.branch }} path: nbc submodules: true fetch-depth: 0 From 72abe822c067ddf3a1315f508c102d995d7625f4 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 15 Sep 2022 09:06:32 +0200 Subject: [PATCH 07/20] Fix switch failed start (#770) --- libp2p/switch.nim | 10 ++++------ tests/testswitch.nim | 9 +++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/libp2p/switch.nim b/libp2p/switch.nim index 3bbdaa6..93f2b5f 100644 --- a/libp2p/switch.nim +++ b/libp2p/switch.nim @@ -312,12 +312,10 @@ proc start*(s: Switch) {.async, gcsafe, public.} = await allFutures(startFuts) - for s in startFuts: - if s.failed: - # TODO: replace this exception with a `listenError` callback. See - # https://github.com/status-im/nim-libp2p/pull/662 for more info. - raise newException(transport.TransportError, - "Failed to start one transport", s.error) + for fut in startFuts: + if fut.failed: + await s.stop() + raise fut.error for t in s.transports: # for each transport if t.addrs.len > 0 or t.running: diff --git a/tests/testswitch.nim b/tests/testswitch.nim index 3d47b1a..608266c 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -1035,3 +1035,12 @@ suite "Switch": await conn.close() await src.stop() await dst.stop() + + asyncTest "switch failing to start stops properly": + let switch = newStandardSwitch( + addrs = @[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet(), MultiAddress.init("/ip4/1.1.1.1/tcp/0").tryGet()] + ) + + expect LPError: + await switch.start() + # test is that this doesn't leak From 5e7e0094455771214ce0cc67f14effbc61ddfd1d Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 15 Sep 2022 09:43:40 +0200 Subject: [PATCH 08/20] Move relay & autonat to connectivity folder (#769) --- examples/circuitrelay.nim | 2 +- libp2p/builders.nim | 4 ++-- .../protocols/{ => connectivity}/autonat.nim | 14 ++++++------- .../{ => connectivity}/relay/client.nim | 8 ++++---- .../{ => connectivity}/relay/messages.nim | 4 ++-- .../{ => connectivity}/relay/rconn.nim | 2 +- .../{ => connectivity}/relay/relay.nim | 20 +++++++++---------- .../{ => connectivity}/relay/rtransport.nim | 6 +++--- .../{ => connectivity}/relay/utils.nim | 2 +- tests/commoninterop.nim | 2 +- tests/testautonat.nim | 2 +- tests/testinterop.nim | 2 +- tests/testrelayv1.nim | 10 +++++----- tests/testrelayv2.nim | 8 ++++---- 14 files changed, 43 insertions(+), 43 deletions(-) rename libp2p/protocols/{ => connectivity}/autonat.nim (98%) rename libp2p/protocols/{ => connectivity}/relay/client.nim (98%) rename libp2p/protocols/{ => connectivity}/relay/messages.nim (99%) rename libp2p/protocols/{ => connectivity}/relay/rconn.nim (98%) rename libp2p/protocols/{ => connectivity}/relay/relay.nim (97%) rename libp2p/protocols/{ => connectivity}/relay/rtransport.nim (97%) rename libp2p/protocols/{ => connectivity}/relay/utils.nim (98%) diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index 290c473..b7c66d2 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -1,6 +1,6 @@ import chronos, stew/byteutils import ../libp2p, - ../libp2p/protocols/relay/[relay, client] + ../libp2p/protocols/connectivity/relay/[relay, client] # Helper to create a circuit relay node proc createCircuitRelaySwitch(r: Relay): Switch = diff --git a/libp2p/builders.nim b/libp2p/builders.nim index 828fc52..5d111b2 100644 --- a/libp2p/builders.nim +++ b/libp2p/builders.nim @@ -26,8 +26,8 @@ import switch, peerid, peerinfo, stream/connection, multiaddress, crypto/crypto, transports/[transport, tcptransport], muxers/[muxer, mplex/mplex, yamux/yamux], - protocols/[identify, secure/secure, secure/noise, autonat], - protocols/relay/[relay, client, rtransport], + protocols/[identify, secure/secure, secure/noise], + protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport], connmanager, upgrademngrs/muxedupgrade, nameresolving/nameresolver, errors, utility diff --git a/libp2p/protocols/autonat.nim b/libp2p/protocols/connectivity/autonat.nim similarity index 98% rename from libp2p/protocols/autonat.nim rename to libp2p/protocols/connectivity/autonat.nim index bdee799..4f7fb53 100644 --- a/libp2p/protocols/autonat.nim +++ b/libp2p/protocols/connectivity/autonat.nim @@ -14,13 +14,13 @@ else: import std/[options, sets, sequtils] import chronos, chronicles, stew/objects -import ./protocol, - ../switch, - ../multiaddress, - ../multicodec, - ../peerid, - ../utils/semaphore, - ../errors +import ../protocol, + ../../switch, + ../../multiaddress, + ../../multicodec, + ../../peerid, + ../../utils/semaphore, + ../../errors logScope: topics = "libp2p autonat" diff --git a/libp2p/protocols/relay/client.nim b/libp2p/protocols/connectivity/relay/client.nim similarity index 98% rename from libp2p/protocols/relay/client.nim rename to libp2p/protocols/connectivity/relay/client.nim index 8f03881..b12a728 100644 --- a/libp2p/protocols/relay/client.nim +++ b/libp2p/protocols/connectivity/relay/client.nim @@ -20,10 +20,10 @@ import ./relay, ./messages, ./rconn, ./utils, - ../../peerinfo, - ../../switch, - ../../multiaddress, - ../../stream/connection + ../../../peerinfo, + ../../../switch, + ../../../multiaddress, + ../../../stream/connection logScope: diff --git a/libp2p/protocols/relay/messages.nim b/libp2p/protocols/connectivity/relay/messages.nim similarity index 99% rename from libp2p/protocols/relay/messages.nim rename to libp2p/protocols/connectivity/relay/messages.nim index 62e2aaa..862d3d9 100644 --- a/libp2p/protocols/relay/messages.nim +++ b/libp2p/protocols/connectivity/relay/messages.nim @@ -14,8 +14,8 @@ else: import options, macros, sequtils import stew/objects -import ../../peerinfo, - ../../signed_envelope +import ../../../peerinfo, + ../../../signed_envelope # Circuit Relay V1 Message diff --git a/libp2p/protocols/relay/rconn.nim b/libp2p/protocols/connectivity/relay/rconn.nim similarity index 98% rename from libp2p/protocols/relay/rconn.nim rename to libp2p/protocols/connectivity/relay/rconn.nim index 44dbb14..f355fd8 100644 --- a/libp2p/protocols/relay/rconn.nim +++ b/libp2p/protocols/connectivity/relay/rconn.nim @@ -14,7 +14,7 @@ else: import chronos -import ../../stream/connection +import ../../../stream/connection type RelayConnection* = ref object of Connection diff --git a/libp2p/protocols/relay/relay.nim b/libp2p/protocols/connectivity/relay/relay.nim similarity index 97% rename from libp2p/protocols/relay/relay.nim rename to libp2p/protocols/connectivity/relay/relay.nim index 10288eb..a19f4c6 100644 --- a/libp2p/protocols/relay/relay.nim +++ b/libp2p/protocols/connectivity/relay/relay.nim @@ -19,16 +19,16 @@ import chronos, chronicles import ./messages, ./rconn, ./utils, - ../../peerinfo, - ../../switch, - ../../multiaddress, - ../../multicodec, - ../../stream/connection, - ../../protocols/protocol, - ../../transports/transport, - ../../errors, - ../../utils/heartbeat, - ../../signed_envelope + ../../../peerinfo, + ../../../switch, + ../../../multiaddress, + ../../../multicodec, + ../../../stream/connection, + ../../../protocols/protocol, + ../../../transports/transport, + ../../../errors, + ../../../utils/heartbeat, + ../../../signed_envelope # TODO: # * Eventually replace std/times by chronos/timer. Currently chronos/timer diff --git a/libp2p/protocols/relay/rtransport.nim b/libp2p/protocols/connectivity/relay/rtransport.nim similarity index 97% rename from libp2p/protocols/relay/rtransport.nim rename to libp2p/protocols/connectivity/relay/rtransport.nim index d84f433..011f682 100644 --- a/libp2p/protocols/relay/rtransport.nim +++ b/libp2p/protocols/connectivity/relay/rtransport.nim @@ -19,9 +19,9 @@ import chronos, chronicles import ./client, ./rconn, ./utils, - ../../switch, - ../../stream/connection, - ../../transports/transport + ../../../switch, + ../../../stream/connection, + ../../../transports/transport logScope: topics = "libp2p relay relay-transport" diff --git a/libp2p/protocols/relay/utils.nim b/libp2p/protocols/connectivity/relay/utils.nim similarity index 98% rename from libp2p/protocols/relay/utils.nim rename to libp2p/protocols/connectivity/relay/utils.nim index efa9744..c5449aa 100644 --- a/libp2p/protocols/relay/utils.nim +++ b/libp2p/protocols/connectivity/relay/utils.nim @@ -17,7 +17,7 @@ import options import chronos, chronicles import ./messages, - ../../stream/connection + ../../../stream/connection logScope: topics = "libp2p relay relay-utils" diff --git a/tests/commoninterop.nim b/tests/commoninterop.nim index c285070..c8fd12b 100644 --- a/tests/commoninterop.nim +++ b/tests/commoninterop.nim @@ -3,7 +3,7 @@ import chronos, chronicles, stew/byteutils import helpers import ../libp2p import ../libp2p/[daemon/daemonapi, varint, transports/wstransport, crypto/crypto] -import ../libp2p/protocols/relay/[relay, client, utils] +import ../libp2p/protocols/connectivity/relay/[relay, client, utils] type SwitchCreator = proc( diff --git a/tests/testautonat.nim b/tests/testautonat.nim index 8d523fa..ce44b58 100644 --- a/tests/testautonat.nim +++ b/tests/testautonat.nim @@ -3,7 +3,7 @@ import chronos import ../libp2p/[ builders, - protocols/autonat + protocols/connectivity/autonat ], ./helpers diff --git a/tests/testinterop.nim b/tests/testinterop.nim index 5cbd53d..8c9f711 100644 --- a/tests/testinterop.nim +++ b/tests/testinterop.nim @@ -2,7 +2,7 @@ import stublogger import helpers, commoninterop import ../libp2p -import ../libp2p/crypto/crypto, ../libp2p/protocols/relay/[relay, client] +import ../libp2p/crypto/crypto, ../libp2p/protocols/connectivity/relay/[relay, client] proc switchMplexCreator( ma: MultiAddress = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet(), diff --git a/tests/testrelayv1.nim b/tests/testrelayv1.nim index 17206be..5595af3 100644 --- a/tests/testrelayv1.nim +++ b/tests/testrelayv1.nim @@ -2,11 +2,11 @@ import options, bearssl, chronos import stew/byteutils -import ../libp2p/[protocols/relay/relay, - protocols/relay/client, - protocols/relay/messages, - protocols/relay/utils, - protocols/relay/rtransport, +import ../libp2p/[protocols/connectivity/relay/relay, + protocols/connectivity/relay/client, + protocols/connectivity/relay/messages, + protocols/connectivity/relay/utils, + protocols/connectivity/relay/rtransport, multiaddress, peerinfo, peerid, diff --git a/tests/testrelayv2.nim b/tests/testrelayv2.nim index 5ce1f64..8565b36 100644 --- a/tests/testrelayv2.nim +++ b/tests/testrelayv2.nim @@ -2,10 +2,10 @@ import bearssl, chronos, options import ../libp2p -import ../libp2p/[protocols/relay/relay, - protocols/relay/messages, - protocols/relay/utils, - protocols/relay/client] +import ../libp2p/[protocols/connectivity/relay/relay, + protocols/connectivity/relay/messages, + protocols/connectivity/relay/utils, + protocols/connectivity/relay/client] import ./helpers import std/times import stew/byteutils From a56c3bc296257c43025eb30f767498825f02d4ee Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Thu, 22 Sep 2022 21:55:59 +0200 Subject: [PATCH 09/20] Make observedAddr optional (#772) Co-authored-by: Tanguy --- libp2p/dial.nim | 5 ++- libp2p/dialer.nim | 5 +-- libp2p/protocols/connectivity/autonat.nim | 18 ++++++++--- libp2p/protocols/identify.nim | 6 ++-- libp2p/protocols/pubsub/pubsubpeer.nim | 5 +-- libp2p/protocols/secure/secure.nim | 5 +-- libp2p/stream/chronosstream.nim | 5 ++- libp2p/stream/connection.nim | 9 +++--- libp2p/transports/tcptransport.nim | 21 ++++++++----- libp2p/transports/wstransport.nim | 10 +++--- tests/commontransport.nim | 8 +++-- tests/testconnmngr.nim | 38 +++++++++++++---------- 12 files changed, 83 insertions(+), 52 deletions(-) diff --git a/libp2p/dial.nim b/libp2p/dial.nim index bb8d00c..e0d78b9 100644 --- a/libp2p/dial.nim +++ b/libp2p/dial.nim @@ -13,10 +13,13 @@ else: {.push raises: [].} import chronos +import stew/results import peerid, stream/connection, transports/transport +export results + type Dial* = ref object of RootObj @@ -69,5 +72,5 @@ method addTransport*( method tryDial*( self: Dial, peerId: PeerId, - addrs: seq[MultiAddress]): Future[MultiAddress] {.async, base.} = + addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} = doAssert(false, "Not implemented!") diff --git a/libp2p/dialer.nim b/libp2p/dialer.nim index 85c5b63..30ff15e 100644 --- a/libp2p/dialer.nim +++ b/libp2p/dialer.nim @@ -9,6 +9,7 @@ import std/[sugar, tables] +import stew/results import pkg/[chronos, chronicles, metrics] @@ -24,7 +25,7 @@ import dial, upgrademngrs/upgrade, errors -export dial, errors +export dial, errors, results logScope: topics = "libp2p dialer" @@ -189,7 +190,7 @@ proc negotiateStream( method tryDial*( self: Dialer, peerId: PeerId, - addrs: seq[MultiAddress]): Future[MultiAddress] {.async.} = + addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} = ## Create a protocol stream in order to check ## if a connection is possible. ## Doesn't use the Connection Manager to save it. diff --git a/libp2p/protocols/connectivity/autonat.nim b/libp2p/protocols/connectivity/autonat.nim index 4f7fb53..6408a7e 100644 --- a/libp2p/protocols/connectivity/autonat.nim +++ b/libp2p/protocols/connectivity/autonat.nim @@ -13,6 +13,7 @@ else: {.push raises: [].} import std/[options, sets, sequtils] +import stew/results import chronos, chronicles, stew/objects import ../protocol, ../../switch, @@ -226,7 +227,10 @@ proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} = try: await a.sem.acquire() let ma = await a.switch.dialer.tryDial(conn.peerId, addrs) - await conn.sendResponseOk(ma) + if ma.isSome: + await conn.sendResponseOk(ma.get()) + else: + await conn.sendResponseError(DialError, "Missing observed address") except CancelledError as exc: raise exc except CatchableError as exc: @@ -241,15 +245,19 @@ proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] = if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId: return conn.sendResponseError(BadRequest, "PeerId mismatch") - var isRelayed = conn.observedAddr.contains(multiCodec("p2p-circuit")) + if conn.observedAddr.isNone: + return conn.sendResponseError(BadRequest, "Missing observed address") + let observedAddr = conn.observedAddr.get() + + var isRelayed = observedAddr.contains(multiCodec("p2p-circuit")) if isRelayed.isErr() or isRelayed.get(): return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address") - let hostIp = conn.observedAddr[0] + let hostIp = observedAddr[0] if hostIp.isErr() or not IP.match(hostIp.get()): - trace "wrong observed address", address=conn.observedAddr + trace "wrong observed address", address=observedAddr return conn.sendResponseError(InternalError, "Expected an IP address") var addrs = initHashSet[MultiAddress]() - addrs.incl(conn.observedAddr) + addrs.incl(observedAddr) for ma in peerInfo.addrs: isRelayed = ma.contains(multiCodec("p2p-circuit")) if isRelayed.isErr() or isRelayed.get(): diff --git a/libp2p/protocols/identify.nim b/libp2p/protocols/identify.nim index 975126f..ae7d26c 100644 --- a/libp2p/protocols/identify.nim +++ b/libp2p/protocols/identify.nim @@ -16,6 +16,7 @@ else: {.push raises: [].} import std/[sequtils, options, strutils, sugar] +import stew/results import chronos, chronicles import ../protobuf/minprotobuf, ../peerinfo, @@ -80,7 +81,7 @@ chronicles.expandIt(IdentifyInfo): if iinfo.signedPeerRecord.isSome(): "Some" else: "None" -proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): ProtoBuffer +proc encodeMsg(peerInfo: PeerInfo, observedAddr: Opt[MultiAddress], sendSpr: bool): ProtoBuffer {.raises: [Defect].} = result = initProtoBuffer() @@ -91,7 +92,8 @@ proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): P result.write(2, ma.data.buffer) for proto in peerInfo.protocols: result.write(3, proto) - result.write(4, observedAddr.data.buffer) + if observedAddr.isSome: + result.write(4, observedAddr.get().data.buffer) let protoVersion = ProtoVersion result.write(5, protoVersion) let agentVersion = if peerInfo.agentVersion.len <= 0: diff --git a/libp2p/protocols/pubsub/pubsubpeer.nim b/libp2p/protocols/pubsub/pubsubpeer.nim index 8af49de..5a1afed 100644 --- a/libp2p/protocols/pubsub/pubsubpeer.nim +++ b/libp2p/protocols/pubsub/pubsubpeer.nim @@ -12,7 +12,8 @@ when (NimMajor, NimMinor) < (1, 4): else: {.push raises: [].} -import std/[sequtils, strutils, tables, hashes] +import std/[sequtils, strutils, tables, hashes, options] +import stew/results import chronos, chronicles, nimcrypto/sha2, metrics import rpc/[messages, message, protobuf], ../../peerid, @@ -174,7 +175,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} = trace "Get new send connection", p, newConn p.sendConn = newConn - p.address = some(p.sendConn.observedAddr) + p.address = if p.sendConn.observedAddr.isSome: some(p.sendConn.observedAddr.get) else: none(MultiAddress) if p.onEvent != nil: p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Connected)) diff --git a/libp2p/protocols/secure/secure.nim b/libp2p/protocols/secure/secure.nim index 0bdf852..c187fd2 100644 --- a/libp2p/protocols/secure/secure.nim +++ b/libp2p/protocols/secure/secure.nim @@ -13,6 +13,7 @@ else: {.push raises: [].} import std/[strformat] +import stew/results import chronos, chronicles import ../protocol, ../../stream/streamseq, @@ -21,7 +22,7 @@ import ../protocol, ../../peerinfo, ../../errors -export protocol +export protocol, results logScope: topics = "libp2p secure" @@ -48,7 +49,7 @@ chronicles.formatIt(SecureConn): shortLog(it) proc new*(T: type SecureConn, conn: Connection, peerId: PeerId, - observedAddr: MultiAddress, + observedAddr: Opt[MultiAddress], timeout: Duration = DefaultConnectionTimeout): T = result = T(stream: conn, peerId: peerId, diff --git a/libp2p/stream/chronosstream.nim b/libp2p/stream/chronosstream.nim index 2b676d4..d123c98 100644 --- a/libp2p/stream/chronosstream.nim +++ b/libp2p/stream/chronosstream.nim @@ -13,10 +13,13 @@ else: {.push raises: [].} import std/[oids, strformat] +import stew/results import chronos, chronicles, metrics import connection import ../utility +export results + logScope: topics = "libp2p chronosstream" @@ -60,7 +63,7 @@ proc init*(C: type ChronosStream, client: StreamTransport, dir: Direction, timeout = DefaultChronosStreamTimeout, - observedAddr: MultiAddress = MultiAddress()): ChronosStream = + observedAddr: Opt[MultiAddress]): ChronosStream = result = C(client: client, timeout: timeout, dir: dir, diff --git a/libp2p/stream/connection.nim b/libp2p/stream/connection.nim index 43f58af..a4f52de 100644 --- a/libp2p/stream/connection.nim +++ b/libp2p/stream/connection.nim @@ -13,13 +13,14 @@ else: {.push raises: [].} import std/[hashes, oids, strformat] +import stew/results import chronicles, chronos, metrics import lpstream, ../multiaddress, ../peerinfo, ../errors -export lpstream, peerinfo, errors +export lpstream, peerinfo, errors, results logScope: topics = "libp2p connection" @@ -37,7 +38,7 @@ type timerTaskFut: Future[void] # the current timer instance timeoutHandler*: TimeoutHandler # timeout handler peerId*: PeerId - observedAddr*: MultiAddress + observedAddr*: Opt[MultiAddress] upgraded*: Future[void] protocol*: string # protocol used by the connection, used as tag for metrics transportDir*: Direction # The bottom level transport (generally the socket) direction @@ -160,9 +161,9 @@ method getWrapped*(s: Connection): Connection {.base.} = proc new*(C: type Connection, peerId: PeerId, dir: Direction, + observedAddr: Opt[MultiAddress], timeout: Duration = DefaultConnectionTimeout, - timeoutHandler: TimeoutHandler = nil, - observedAddr: MultiAddress = MultiAddress()): Connection = + timeoutHandler: TimeoutHandler = nil): Connection = result = C(peerId: peerId, dir: dir, timeout: timeout, diff --git a/libp2p/transports/tcptransport.nim b/libp2p/transports/tcptransport.nim index 24f63f6..6d0d321 100644 --- a/libp2p/transports/tcptransport.nim +++ b/libp2p/transports/tcptransport.nim @@ -15,6 +15,7 @@ else: {.push raises: [].} import std/[oids, sequtils] +import stew/results import chronos, chronicles import transport, ../errors, @@ -31,7 +32,7 @@ import transport, logScope: topics = "libp2p tcptransport" -export transport +export transport, results const TcpTransportTrackerName* = "libp2p.tcptransport" @@ -71,18 +72,20 @@ proc setupTcpTransportTracker(): TcpTransportTracker = result.isLeaked = leakTransport addTracker(TcpTransportTrackerName, result) -proc connHandler*(self: TcpTransport, - client: StreamTransport, - dir: Direction): Future[Connection] {.async.} = - var observedAddr: MultiAddress = MultiAddress() +proc getObservedAddr(client: StreamTransport): Future[MultiAddress] {.async.} = try: - observedAddr = MultiAddress.init(client.remoteAddress).tryGet() + return MultiAddress.init(client.remoteAddress).tryGet() except CatchableError as exc: trace "Failed to create observedAddr", exc = exc.msg if not(isNil(client) and client.closed): await client.closeWait() raise exc +proc connHandler*(self: TcpTransport, + client: StreamTransport, + observedAddr: Opt[MultiAddress], + dir: Direction): Future[Connection] {.async.} = + trace "Handling tcp connection", address = $observedAddr, dir = $dir, clients = self.clients[Direction.In].len + @@ -222,7 +225,8 @@ method accept*(self: TcpTransport): Future[Connection] {.async, gcsafe.} = self.acceptFuts[index] = self.servers[index].accept() let transp = await finished - return await self.connHandler(transp, Direction.In) + let observedAddr = await getObservedAddr(transp) + return await self.connHandler(transp, Opt.some(observedAddr), Direction.In) except TransportOsError as exc: # TODO: it doesn't sound like all OS errors # can be ignored, we should re-raise those @@ -250,7 +254,8 @@ method dial*( let transp = await connect(address) try: - return await self.connHandler(transp, Direction.Out) + let observedAddr = await getObservedAddr(transp) + return await self.connHandler(transp, Opt.some(observedAddr), Direction.Out) except CatchableError as err: await transp.closeWait() raise err diff --git a/libp2p/transports/wstransport.nim b/libp2p/transports/wstransport.nim index 50eb7c3..9ded1c1 100644 --- a/libp2p/transports/wstransport.nim +++ b/libp2p/transports/wstransport.nim @@ -15,6 +15,7 @@ else: {.push raises: [].} import std/[sequtils] +import stew/results import chronos, chronicles import transport, ../errors, @@ -31,7 +32,7 @@ import transport, logScope: topics = "libp2p wstransport" -export transport, websock +export transport, websock, results const WsTransportTrackerName* = "libp2p.wstransport" @@ -45,8 +46,8 @@ type proc new*(T: type WsStream, session: WSSession, dir: Direction, - timeout = 10.minutes, - observedAddr: MultiAddress = MultiAddress()): T = + observedAddr: Opt[MultiAddress], + timeout = 10.minutes): T = let stream = T( session: session, @@ -221,8 +222,7 @@ proc connHandler(self: WsTransport, await stream.close() raise exc - let conn = WsStream.new(stream, dir) - conn.observedAddr = observedAddr + let conn = WsStream.new(stream, dir, Opt.some(observedAddr)) self.connections[dir].add(conn) proc onClose() {.async.} = diff --git a/tests/commontransport.nim b/tests/commontransport.nim index 532689e..af29add 100644 --- a/tests/commontransport.nim +++ b/tests/commontransport.nim @@ -1,7 +1,7 @@ {.used.} import sequtils -import chronos, stew/byteutils +import chronos, stew/[byteutils, results] import ../libp2p/[stream/connection, transports/transport, upgrademngrs/upgrade, @@ -35,14 +35,16 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) = proc acceptHandler() {.async, gcsafe.} = let conn = await transport1.accept() - check transport1.handles(conn.observedAddr) + if conn.observedAddr.isSome(): + check transport1.handles(conn.observedAddr.get()) await conn.close() let handlerWait = acceptHandler() let conn = await transport2.dial(transport1.addrs[0]) - check transport2.handles(conn.observedAddr) + if conn.observedAddr.isSome(): + check transport2.handles(conn.observedAddr.get()) await conn.close() #for some protocols, closing requires actively reading, so we must close here diff --git a/tests/testconnmngr.nim b/tests/testconnmngr.nim index 5c705bf..dda2e3d 100644 --- a/tests/testconnmngr.nim +++ b/tests/testconnmngr.nim @@ -1,4 +1,5 @@ import sequtils +import stew/results import chronos import ../libp2p/[connmanager, stream/connection, @@ -9,6 +10,9 @@ import ../libp2p/[connmanager, import helpers +proc getConnection(peerId: PeerId, dir: Direction = Direction.In): Connection = + return Connection.new(peerId, dir, Opt.none(MultiAddress)) + type TestMuxer = ref object of Muxer peerId: PeerId @@ -18,7 +22,7 @@ method newStream*( name: string = "", lazy: bool = false): Future[Connection] {.async, gcsafe.} = - result = Connection.new(m.peerId, Direction.Out) + result = getConnection(m.peerId, Direction.Out) suite "Connection Manager": teardown: @@ -27,7 +31,7 @@ suite "Connection Manager": asyncTest "add and retrieve a connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) connMngr.storeConn(conn) check conn in connMngr @@ -41,7 +45,7 @@ suite "Connection Manager": asyncTest "shouldn't allow a closed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) await conn.close() expect CatchableError: @@ -52,7 +56,7 @@ suite "Connection Manager": asyncTest "shouldn't allow an EOFed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) conn.isEof = true expect CatchableError: @@ -64,7 +68,7 @@ suite "Connection Manager": asyncTest "add and retrieve a muxer": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -80,7 +84,7 @@ suite "Connection Manager": asyncTest "shouldn't allow a muxer for an untracked connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -94,8 +98,8 @@ suite "Connection Manager": asyncTest "get conn with direction": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn1 = Connection.new(peerId, Direction.Out) - let conn2 = Connection.new(peerId, Direction.In) + let conn1 = getConnection(peerId, Direction.Out) + let conn2 = getConnection(peerId) connMngr.storeConn(conn1) connMngr.storeConn(conn2) @@ -114,7 +118,7 @@ suite "Connection Manager": asyncTest "get muxed stream for peer": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -134,7 +138,7 @@ suite "Connection Manager": asyncTest "get stream from directed connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -155,7 +159,7 @@ suite "Connection Manager": asyncTest "get stream from any connection": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new TestMuxer muxer.peerId = peerId @@ -175,11 +179,11 @@ suite "Connection Manager": let connMngr = ConnManager.new(maxConnsPerPeer = 1) let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - connMngr.storeConn(Connection.new(peerId, Direction.In)) + connMngr.storeConn(getConnection(peerId)) let conns = @[ - Connection.new(peerId, Direction.In), - Connection.new(peerId, Direction.In)] + getConnection(peerId), + getConnection(peerId)] expect TooManyConnectionsError: connMngr.storeConn(conns[0]) @@ -193,7 +197,7 @@ suite "Connection Manager": asyncTest "cleanup on connection close": let connMngr = ConnManager.new() let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet() - let conn = Connection.new(peerId, Direction.In) + let conn = getConnection(peerId) let muxer = new Muxer muxer.connection = conn @@ -220,7 +224,7 @@ suite "Connection Manager": Direction.In else: Direction.Out - let conn = Connection.new(peerId, dir) + let conn = getConnection(peerId, dir) let muxer = new Muxer muxer.connection = conn @@ -353,7 +357,7 @@ suite "Connection Manager": let slot = await ((connMngr.getOutgoingSlot()).wait(10.millis)) let conn = - Connection.new( + getConnection( PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet(), Direction.In) From fa5d102370cff4e68b47ac633153b22609b8aa41 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 26 Sep 2022 11:03:24 +0200 Subject: [PATCH 10/20] Better dnsaddr resolving (#753) --- libp2p/dialer.nim | 134 ++++++++++++++++++-------- libp2p/multiaddress.nim | 24 ++++- libp2p/nameresolving/nameresolver.nim | 64 ++++++------ tests/testmultiaddress.nim | 5 + tests/testnameresolve.nim | 17 +++- tests/testnoise.nim | 2 +- tests/testswitch.nim | 23 ++++- 7 files changed, 177 insertions(+), 92 deletions(-) diff --git a/libp2p/dialer.nim b/libp2p/dialer.nim index 30ff15e..d3c0826 100644 --- a/libp2p/dialer.nim +++ b/libp2p/dialer.nim @@ -7,7 +7,7 @@ # This file may not be copied, modified, or distributed except according to # those terms. -import std/[sugar, tables] +import std/[sugar, tables, sequtils] import stew/results import pkg/[chronos, @@ -17,6 +17,7 @@ import pkg/[chronos, import dial, peerid, peerinfo, + multicodec, multistream, connmanager, stream/connection, @@ -46,56 +47,105 @@ type transports: seq[Transport] nameResolver: NameResolver +proc dialAndUpgrade( + self: Dialer, + peerId: Opt[PeerId], + hostname: string, + address: MultiAddress): + Future[Connection] {.async.} = + + for transport in self.transports: # for each transport + if transport.handles(address): # check if it can dial it + trace "Dialing address", address, peerId, hostname + let dialed = + try: + libp2p_total_dial_attempts.inc() + await transport.dial(hostname, address) + except CancelledError as exc: + debug "Dialing canceled", msg = exc.msg, peerId + raise exc + except CatchableError as exc: + debug "Dialing failed", msg = exc.msg, peerId + libp2p_failed_dials.inc() + return nil # Try the next address + + # also keep track of the connection's bottom unsafe transport direction + # required by gossipsub scoring + dialed.transportDir = Direction.Out + + libp2p_successful_dials.inc() + + let conn = + try: + await transport.upgradeOutgoing(dialed, peerId) + except CatchableError as exc: + # If we failed to establish the connection through one transport, + # we won't succeeded through another - no use in trying again + await dialed.close() + debug "Upgrade failed", msg = exc.msg, peerId + if exc isnot CancelledError: + libp2p_failed_upgrades_outgoing.inc() + + # Try other address + return nil + + doAssert not isNil(conn), "connection died after upgradeOutgoing" + debug "Dial successful", conn, peerId = conn.peerId + return conn + return nil + +proc expandDnsAddr( + self: Dialer, + peerId: Opt[PeerId], + address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} = + + if not DNSADDR.matchPartial(address): return @[(address, peerId)] + if isNil(self.nameResolver): + info "Can't resolve DNSADDR without NameResolver", ma=address + return @[] + + let + toResolve = + if peerId.isSome: + address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet() + else: + address + resolved = await self.nameResolver.resolveDnsAddr(toResolve) + + for resolvedAddress in resolved: + let lastPart = resolvedAddress[^1].tryGet() + if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")): + let + peerIdBytes = lastPart.protoArgument().tryGet() + addrPeerId = PeerId.init(peerIdBytes).tryGet() + result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId))) + else: + result.add((resolvedAddress, peerId)) + proc dialAndUpgrade( self: Dialer, peerId: Opt[PeerId], addrs: seq[MultiAddress]): Future[Connection] {.async.} = + debug "Dialing peer", peerId - for address in addrs: # for each address - let - hostname = address.getHostname() - resolvedAddresses = - if isNil(self.nameResolver): @[address] - else: await self.nameResolver.resolveMAddress(address) + for rawAddress in addrs: + # resolve potential dnsaddr + let addresses = await self.expandDnsAddr(peerId, rawAddress) - for a in resolvedAddresses: # for each resolved address - for transport in self.transports: # for each transport - if transport.handles(a): # check if it can dial it - trace "Dialing address", address = $a, peerId, hostname - let dialed = try: - libp2p_total_dial_attempts.inc() - await transport.dial(hostname, a) - except CancelledError as exc: - debug "Dialing canceled", msg = exc.msg, peerId - raise exc - except CatchableError as exc: - debug "Dialing failed", msg = exc.msg, peerId - libp2p_failed_dials.inc() - continue # Try the next address + for (expandedAddress, addrPeerId) in addresses: + # DNS resolution + let + hostname = expandedAddress.getHostname() + resolvedAddresses = + if isNil(self.nameResolver): @[expandedAddress] + else: await self.nameResolver.resolveMAddress(expandedAddress) - # also keep track of the connection's bottom unsafe transport direction - # required by gossipsub scoring - dialed.transportDir = Direction.Out - - libp2p_successful_dials.inc() - - let conn = try: - await transport.upgradeOutgoing(dialed, peerId) - except CatchableError as exc: - # If we failed to establish the connection through one transport, - # we won't succeeded through another - no use in trying again - # TODO we should try another address though - await dialed.close() - debug "Upgrade failed", msg = exc.msg, peerId - if exc isnot CancelledError: - libp2p_failed_upgrades_outgoing.inc() - raise exc - - doAssert not isNil(conn), "connection died after upgradeOutgoing" - debug "Dial successful", conn, peerId = conn.peerId - return conn + for resolvedAddress in resolvedAddresses: + result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress) + if not isNil(result): + return result proc internalConnect( self: Dialer, diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index 054fb00..fd663f9 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -573,7 +573,7 @@ proc protoArgument*(ma: MultiAddress, err("multiaddress: Decoding protocol error") else: ok(res) - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(buffer) == -1: err("multiaddress: Decoding protocol error") else: @@ -594,6 +594,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] = buffer.setLen(res) ok(buffer) +proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] = + ## Returns MultiAddress ``ma`` protocol address binary blob. + ## + ## If current MultiAddress do not have argument value, then result array will + ## be empty. + ma.protoAddress() + proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = var header: uint64 var data = newSeq[byte]() @@ -601,6 +608,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = var vb = ma var res: MultiAddress res.data = initVBuffer() + + if index < 0: return err("multiaddress: negative index gived to getPart") + while offset <= index: if vb.data.readVarint(header) == -1: return err("multiaddress: Malformed binary address!") @@ -618,7 +628,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] = res.data.writeVarint(header) res.data.writeArray(data) res.data.finish() - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(data) == -1: return err("multiaddress: Decoding protocol error") @@ -647,9 +657,13 @@ proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddres ? res.append(? ma[i]) ok(res) -proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} = +proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} = ## Returns part with index ``i`` of MultiAddress ``ma``. - ma.getPart(i) + when i is BackwardsIndex: + let maLength = ? len(ma) + ma.getPart(maLength - int(i)) + else: + ma.getPart(i) proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} = ## Returns parts with slice ``slice`` of MultiAddress ``ma``. @@ -680,7 +694,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] = res.data.writeVarint(header) res.data.writeArray(data) - elif proto.kind in {Length, Path}: + elif proto.kind in {MAKind.Length, Path}: if vb.data.readSeq(data) == -1: yield err(MaResult[MultiAddress], "Decoding protocol error") diff --git a/libp2p/nameresolving/nameresolver.nim b/libp2p/nameresolving/nameresolver.nim index 0fdd6a0..b53e0b4 100644 --- a/libp2p/nameresolving/nameresolver.nim +++ b/libp2p/nameresolving/nameresolver.nim @@ -13,7 +13,7 @@ else: {.push raises: [].} import std/[sugar, sets, sequtils, strutils] -import +import chronos, chronicles, stew/[endians2, byteutils] @@ -22,14 +22,14 @@ import ".."/[multiaddress, multicodec] logScope: topics = "libp2p nameresolver" -type +type NameResolver* = ref object of RootObj method resolveTxt*( self: NameResolver, address: string): Future[seq[string]] {.async, base.} = ## Get TXT record - ## + ## doAssert(false, "Not implemented!") @@ -39,16 +39,18 @@ method resolveIp*( port: Port, domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async, base.} = ## Resolve the specified address - ## + ## doAssert(false, "Not implemented!") proc getHostname*(ma: MultiAddress): string = - let firstPart = ($ma[0].get()).split('/') - if firstPart.len > 1: firstPart[2] + let + firstPart = ma[0].valueOr: return "" + fpSplitted = ($firstPart).split('/', 2) + if fpSplitted.len > 2: fpSplitted[2] else: "" -proc resolveDnsAddress( +proc resolveOneAddress( self: NameResolver, ma: MultiAddress, domain: Domain = Domain.AF_UNSPEC, @@ -64,29 +66,22 @@ proc resolveDnsAddress( let port = Port(fromBytesBE(uint16, pbuf)) resolvedAddresses = await self.resolveIp(prefix & dnsval, port, domain) - + return collect(newSeqOfCap(4)): for address in resolvedAddresses: var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet() for part in ma: - if DNS.match(part.get()): continue + if DNS.match(part.tryGet()): continue createdAddress &= part.tryGet() createdAddress -func matchDnsSuffix(m1, m2: MultiAddress): MaResult[bool] = - for partMaybe in m1: - let part = ?partMaybe - if DNS.match(part): continue - let entryProt = ?m2[?part.protoCode()] - if entryProt != part: - return ok(false) - return ok(true) - -proc resolveDnsAddr( +proc resolveDnsAddr*( self: NameResolver, ma: MultiAddress, - depth: int = 0): Future[seq[MultiAddress]] - {.async.} = + depth: int = 0): Future[seq[MultiAddress]] {.async.} = + + if not DNSADDR.matchPartial(ma): + return @[ma] trace "Resolving dnsaddr", ma if depth > 6: @@ -104,21 +99,17 @@ proc resolveDnsAddr( if not entry.startsWith("dnsaddr="): continue let entryValue = MultiAddress.init(entry[8..^1]).tryGet() - if not matchDnsSuffix(ma, entryValue).tryGet(): continue + if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet(): + if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]: + continue - # The spec is not clear wheter only DNSADDR can be recursived - # or any DNS addr. Only handling DNSADDR because it's simpler - # to avoid infinite recursion - if DNSADDR.matchPartial(entryValue): - let resolved = await self.resolveDnsAddr(entryValue, depth + 1) - for r in resolved: - result.add(r) - else: - result.add(entryValue) + let resolved = await self.resolveDnsAddr(entryValue, depth + 1) + for r in resolved: + result.add(r) if result.len == 0: - debug "Failed to resolve any DNSADDR", ma - return @[ma] + debug "Failed to resolve a DNSADDR", ma + return @[] return result @@ -133,14 +124,15 @@ proc resolveMAddress*( let code = address[0].get().protoCode().get() let seq = case code: of multiCodec("dns"): - await self.resolveDnsAddress(address) + await self.resolveOneAddress(address) of multiCodec("dns4"): - await self.resolveDnsAddress(address, Domain.AF_INET) + await self.resolveOneAddress(address, Domain.AF_INET) of multiCodec("dns6"): - await self.resolveDnsAddress(address, Domain.AF_INET6) + await self.resolveOneAddress(address, Domain.AF_INET6) of multiCodec("dnsaddr"): await self.resolveDnsAddr(address) else: + doAssert false @[address] for ad in seq: res.incl(ad) diff --git a/tests/testmultiaddress.nim b/tests/testmultiaddress.nim index 2c05e21..c180db7 100644 --- a/tests/testmultiaddress.nim +++ b/tests/testmultiaddress.nim @@ -386,6 +386,11 @@ suite "MultiAddress test suite": let ma = MultiAddress.init("/ip4/0.0.0.0/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio/").get() check: $ma[0..0].get() == "/ip4/0.0.0.0" + $ma[^1].get() == "/unix/stdio" + ma[-100].isErr() + ma[100].isErr() + ma[^100].isErr() + ma[^0].isErr() $ma[0..1].get() == "/ip4/0.0.0.0/tcp/0" $ma[1..2].get() == "/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" $ma[^3..^1].get() == "/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio" diff --git a/tests/testnameresolve.nim b/tests/testnameresolve.nim index fa9b88f..4a4dd40 100644 --- a/tests/testnameresolve.nim +++ b/tests/testnameresolve.nim @@ -139,7 +139,18 @@ suite "Name resolving": asyncTest "dnsaddr infinite recursion": resolver.txtResponses["_dnsaddr.bootstrap.libp2p.io"] = @["dnsaddr=/dnsaddr/bootstrap.libp2p.io"] - check testOne("/dnsaddr/bootstrap.libp2p.io/", "/dnsaddr/bootstrap.libp2p.io/") + check testOne("/dnsaddr/bootstrap.libp2p.io/", newSeq[string]()) + + test "getHostname": + check: + MultiAddress.init("/dnsaddr/bootstrap.libp2p.io/").tryGet().getHostname == "bootstrap.libp2p.io" + MultiAddress.init("").tryGet().getHostname == "" + MultiAddress.init("/ip4/147.75.69.143/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "147.75.69.143" + MultiAddress.init("/ip6/2604:1380:1000:6000::1/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "2604:1380:1000:6000::1" + MultiAddress.init("/dns/localhost/udp/0").tryGet().getHostname == "localhost" + MultiAddress.init("/dns4/hello.com/udp/0").tryGet().getHostname == "hello.com" + MultiAddress.init("/dns6/hello.com/udp/0").tryGet().getHostname == "hello.com" + MultiAddress.init("/wss/").tryGet().getHostname == "" suite "DNS Resolving": teardown: @@ -171,7 +182,7 @@ suite "Name resolving": # The test var dnsresolver = DnsResolver.new(@[server.localAddress]) - + check await(dnsresolver.resolveIp("status.im", 0.Port, Domain.AF_UNSPEC)) == mapIt( @["104.22.24.181:0", "172.67.10.161:0", "104.22.25.181:0", @@ -209,7 +220,7 @@ suite "Name resolving": # The test var dnsresolver = DnsResolver.new(@[unresponsiveServer.localAddress, server.localAddress]) - + check await(dnsresolver.resolveIp("status.im", 0.Port, Domain.AF_INET)) == mapIt(@["104.22.24.181:0", "172.67.10.161:0", "104.22.25.181:0"], initTAddress(it)) diff --git a/tests/testnoise.nim b/tests/testnoise.nim index b0e785b..a94714c 100644 --- a/tests/testnoise.nim +++ b/tests/testnoise.nim @@ -295,7 +295,7 @@ suite "Noise": (switch2, peerInfo2) = createSwitch(ma2, true, true) # secio, we want to fail await switch1.start() await switch2.start() - expect(UpgradeFailedError): + expect(DialFailedError): let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) await allFuturesThrowing( diff --git a/tests/testswitch.nim b/tests/testswitch.nim index 608266c..961b6dc 100644 --- a/tests/testswitch.nim +++ b/tests/testswitch.nim @@ -201,13 +201,25 @@ suite "Switch": check not switch1.isConnected(switch2.peerInfo.peerId) check not switch2.isConnected(switch1.peerInfo.peerId) - asyncTest "e2e connect to peer with unkown PeerId": + asyncTest "e2e connect to peer with unknown PeerId": + let resolver = MockResolver.new() let switch1 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) - let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise]) + let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise], nameResolver = resolver) await switch1.start() await switch2.start() + # via dnsaddr + resolver.txtResponses["_dnsaddr.test.io"] = @[ + "dnsaddr=" & $switch1.peerInfo.addrs[0] & "/p2p/" & $switch1.peerInfo.peerId, + ] + + check: (await switch2.connect(@[MultiAddress.init("/dnsaddr/test.io/").tryGet()])) == switch1.peerInfo.peerId + await switch2.disconnect(switch1.peerInfo.peerId) + + # via direct ip + check not switch2.isConnected(switch1.peerInfo.peerId) check: (await switch2.connect(switch1.peerInfo.addrs)) == switch1.peerInfo.peerId + await switch2.disconnect(switch1.peerInfo.peerId) await allFuturesThrowing( @@ -665,7 +677,7 @@ suite "Switch": await switch.start() var peerId = PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).get() - expect LPStreamClosedError, LPStreamEOFError: + expect DialFailedError: await switch.connect(peerId, transport.addrs) await handlerWait @@ -994,9 +1006,10 @@ suite "Switch": await srcWsSwitch.start() resolver.txtResponses["_dnsaddr.test.io"] = @[ - "dnsaddr=" & $destSwitch.peerInfo.addrs[0], - "dnsaddr=" & $destSwitch.peerInfo.addrs[1] + "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[0][1..^1].tryGet() & "/p2p/" & $destSwitch.peerInfo.peerId, + "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[1][1..^1].tryGet() ] + resolver.ipResponses[("localhost", false)] = @["127.0.0.1"] let testAddr = MultiAddress.init("/dnsaddr/test.io/").tryGet() From eb786607023e0773f0d2919d68716034a394b052 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 28 Sep 2022 10:40:53 +0200 Subject: [PATCH 11/20] Docs rework (#776) --- .github/workflows/doc.yml | 8 ++- .gitignore | 1 + examples/README.md | 2 +- examples/circuitrelay.nim | 12 +++- examples/directchat.nim | 2 +- examples/helloworld.nim | 2 +- examples/tutorial_1_connect.md | 108 ---------------------------- examples/tutorial_1_connect.nim | 95 ++++++++++++++++++++++++ examples/tutorial_2_customproto.md | 82 --------------------- examples/tutorial_2_customproto.nim | 73 +++++++++++++++++++ libp2p.nimble | 18 +++-- mkdocs.yml | 9 ++- tools/markdown_builder.nim | 29 ++++++++ 13 files changed, 235 insertions(+), 206 deletions(-) delete mode 100644 examples/tutorial_1_connect.md create mode 100644 examples/tutorial_1_connect.nim delete mode 100644 examples/tutorial_2_customproto.md create mode 100644 examples/tutorial_2_customproto.nim create mode 100644 tools/markdown_builder.nim diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 862c571..dce8676 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -63,7 +63,7 @@ jobs: git push origin gh-pages update_site: - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs' name: 'Rebuild website' runs-on: ubuntu-latest steps: @@ -74,8 +74,12 @@ jobs: with: python-version: 3.x + - uses: jiro4989/setup-nim-action@v1 + with: + nim-version: 'stable' + - name: Generate website - run: pip install mkdocs-material && mkdocs build + run: pip install mkdocs-material && nimble website - name: Clone the gh-pages branch uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index ec585b0..f93a08d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,6 @@ build/ .vscode/ .DS_Store tests/pubsub/testgossipsub +examples/*.md nimble.develop nimble.paths diff --git a/examples/README.md b/examples/README.md index 18b6ca8..0760073 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,5 +2,5 @@ Welcome to the nim-libp2p documentation! -Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and +Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html). diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index b7c66d2..bf90d97 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -1,6 +1,14 @@ +## # Circuit Relay example +## +## Circuit Relay can be used when a node cannot reach another node +## directly, but can reach it through a another node (the Relay). +## +## That may happen because of NAT, Firewalls, or incompatible transports. +## +## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/). import chronos, stew/byteutils -import ../libp2p, - ../libp2p/protocols/connectivity/relay/[relay, client] +import libp2p, + libp2p/protocols/connectivity/relay/[relay, client] # Helper to create a circuit relay node proc createCircuitRelaySwitch(r: Relay): Switch = diff --git a/examples/directchat.nim b/examples/directchat.nim index 9e7d99c..b550d48 100644 --- a/examples/directchat.nim +++ b/examples/directchat.nim @@ -5,7 +5,7 @@ import strformat, strutils, stew/byteutils, chronos, - ../libp2p + libp2p const DefaultAddr = "/ip4/127.0.0.1/tcp/0" diff --git a/examples/helloworld.nim b/examples/helloworld.nim index 9f5f1cb..7f144aa 100644 --- a/examples/helloworld.nim +++ b/examples/helloworld.nim @@ -1,6 +1,6 @@ import chronos # an efficient library for async import stew/byteutils # various utils -import ../libp2p # when installed through nimble, just use `import libp2p` +import libp2p ## # Create our custom protocol diff --git a/examples/tutorial_1_connect.md b/examples/tutorial_1_connect.md deleted file mode 100644 index 0dd6543..0000000 --- a/examples/tutorial_1_connect.md +++ /dev/null @@ -1,108 +0,0 @@ -# Simple ping tutorial - -Hi all, welcome to the first nim-libp2p tutorial! - -!!! tips "" - This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. - -To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. - -Hope you'll find it helpful in your journey of learning. Happy coding! ;) - -## Before you start -The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). - -Install Nim via their [official website](https://nim-lang.org/install.html). -Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. - -You can now install the latest version of `nim-libp2p`: -```bash -nimble install libp2p@#master -``` - -## A simple ping application -We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. - -!!! tips "" - You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder! - -Let's create a `part1.nim`, and import our dependencies: -```nim -import chronos - -import libp2p -import libp2p/protocols/ping -``` -[chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` - -Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: -```nim -proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = - var switch = SwitchBuilder - .new() - .withRng(rng) # Give the application RNG - .withAddress(ma) # Our local address(es) - .withTcpTransport() # Use TCP as transport - .withMplex() # Use Mplex as muxer - .withNoise() # Use Noise as secure manager - .build() - - return switch -``` -This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. - -You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. - - -Let's now start to create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() - pingProtocol = Ping.new(rng=rng) -``` -We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. -The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". - -`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. - -We can now create our two switches: -```nim - let - switch1 = createSwitch(localAddress, rng) - switch2 = createSwitch(localAddress, rng) - - switch1.mount(pingProtocol) - - await switch1.start() - await switch2.start() -``` -We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. - -Now that we've started the nodes, they are listening for incoming peers. -We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. - -We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: -```nim - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) -``` -We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: -```nim - # ping the other node and echo the ping duration - echo "ping: ", await pingProtocol.ping(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() -``` - -And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: -```nim - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports - -waitFor(main()) -``` - -You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. - -In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_1_connect.nim b/examples/tutorial_1_connect.nim new file mode 100644 index 0000000..d8d8c2b --- /dev/null +++ b/examples/tutorial_1_connect.nim @@ -0,0 +1,95 @@ +## # Simple ping tutorial +## +## Hi all, welcome to the first nim-libp2p tutorial! +## +## !!! tips "" +## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. +## +## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. +## +## Hope you'll find it helpful in your journey of learning. Happy coding! ;) +## +## ## Before you start +## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). +## +## Install Nim via their [official website](https://nim-lang.org/install.html). +## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. +## +## You can now install the latest version of `nim-libp2p`: +## ```bash +## nimble install libp2p@#master +## ``` +## +## ## A simple ping application +## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. +## +## !!! tips "" +## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder! +## +## Let's create a `part1.nim`, and import our dependencies: +import chronos + +import libp2p +import libp2p/protocols/ping + +## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` +## +## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: +proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = + var switch = SwitchBuilder + .new() + .withRng(rng) # Give the application RNG + .withAddress(ma) # Our local address(es) + .withTcpTransport() # Use TCP as transport + .withMplex() # Use Mplex as muxer + .withNoise() # Use Noise as secure manager + .build() + + return switch + +## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. +## +## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. +## +## +## Let's now start to create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() + pingProtocol = Ping.new(rng=rng) + ## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. + ## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". + ## + ## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. + ## + ## We can now create our two switches: + let + switch1 = createSwitch(localAddress, rng) + switch2 = createSwitch(localAddress, rng) + + switch1.mount(pingProtocol) + + await switch1.start() + await switch2.start() + ## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. + ## + ## Now that we've started the nodes, they are listening for incoming peers. + ## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. + ## + ## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) + ## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: + # ping the other node and echo the ping duration + echo "ping: ", await pingProtocol.ping(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + ## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +waitFor(main()) + +## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. +## +## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_2_customproto.md b/examples/tutorial_2_customproto.md deleted file mode 100644 index aa3366c..0000000 --- a/examples/tutorial_2_customproto.md +++ /dev/null @@ -1,82 +0,0 @@ -# Custom protocol in libp2p - -In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. - -We'll now look at how to create a custom protocol inside the libp2p - -Let's create a `part2.nim`, and import our dependencies: -```nim -import chronos -import stew/byteutils - -import libp2p -``` -This is similar to the first tutorial, except we don't need to import the `Ping` protocol. - -Next, we'll declare our custom protocol -```nim -const TestCodec = "/test/proto/1.0.0" - -type TestProto = ref object of LPProtocol -``` - -We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. - -A protocol generally has two part: and handling/server part, and a dialing/client part. -Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. - -Let's start with the server part: -```nim -proc new(T: typedesc[TestProto]): T = - # every incoming connections will in be handled in this closure - proc handle(conn: Connection, proto: string) {.async, gcsafe.} = - # Read up to 1024 bytes from this connection, and transform them into - # a string - echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) - # We must close the connections ourselves when we're done with it - await conn.close() - - return T(codecs: @[TestCodec], handler: handle) -``` -This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. -In our handle, we simply read a message from the connection and `echo` it. - -We can now create our client part: -```nim -proc hello(p: TestProto, conn: Connection) {.async.} = - await conn.writeLp("Hello p2p!") -``` -Again, pretty straight-forward, we just send a message on the connection. - -We can now create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - testProto = TestProto.new() - switch1 = newStandardSwitch(rng=rng) - switch2 = newStandardSwitch(rng=rng) - - switch1.mount(testProto) - - await switch1.start() - await switch2.start() - - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) - - await testProto.hello(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() - - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports -``` - -This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p - -We can now wrap our program by calling our main proc: -```nim -waitFor(main()) -``` - -And that's it! diff --git a/examples/tutorial_2_customproto.nim b/examples/tutorial_2_customproto.nim new file mode 100644 index 0000000..2f5789b --- /dev/null +++ b/examples/tutorial_2_customproto.nim @@ -0,0 +1,73 @@ +## # Custom protocol in libp2p +## +## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. +## +## We'll now look at how to create a custom protocol inside the libp2p +## +## Let's create a `part2.nim`, and import our dependencies: +import chronos +import stew/byteutils + +import libp2p +## This is similar to the first tutorial, except we don't need to import the `Ping` protocol. +## +## Next, we'll declare our custom protocol +const TestCodec = "/test/proto/1.0.0" + +type TestProto = ref object of LPProtocol + +## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. +## +## A protocol generally has two part: and handling/server part, and a dialing/client part. +## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. +## +## Let's start with the server part: + +proc new(T: typedesc[TestProto]): T = + # every incoming connections will in be handled in this closure + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + # Read up to 1024 bytes from this connection, and transform them into + # a string + echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) + # We must close the connections ourselves when we're done with it + await conn.close() + + return T(codecs: @[TestCodec], handler: handle) + +## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. +## In our handle, we simply read a message from the connection and `echo` it. +## +## We can now create our client part: +proc hello(p: TestProto, conn: Connection) {.async.} = + await conn.writeLp("Hello p2p!") + +## Again, pretty straight-forward, we just send a message on the connection. +## +## We can now create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + testProto = TestProto.new() + switch1 = newStandardSwitch(rng=rng) + switch2 = newStandardSwitch(rng=rng) + + switch1.mount(testProto) + + await switch1.start() + await switch2.start() + + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) + + await testProto.hello(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p +## +## We can now wrap our program by calling our main proc: +waitFor(main()) + +## And that's it! diff --git a/libp2p.nimble b/libp2p.nimble index fb2ef48..a1c3b04 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -32,16 +32,16 @@ proc runTest(filename: string, verify: bool = true, sign: bool = true, rmFile "tests/" & filename.toExe proc buildSample(filename: string, run = false) = - var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off " + var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " excstr.add(" examples/" & filename) exec excstr if run: exec "./examples/" & filename.toExe rmFile "examples/" & filename.toExe -proc buildTutorial(filename: string) = - discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " & - " nim --verbosity:0 --hints:off c -" +proc tutorialToMd(filename: string) = + let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim " + writeFile(filename.replace(".nim", ".md"), markdown) task testnative, "Runs libp2p native tests": runTest("testnative") @@ -86,12 +86,18 @@ task test_slim, "Runs the (slimmed down) test suite": exec "nimble testfilter" exec "nimble examples_build" +task website, "Build the website": + tutorialToMd("examples/tutorial_1_connect.nim") + tutorialToMd("examples/tutorial_2_customproto.nim") + tutorialToMd("examples/circuitrelay.nim") + exec "mkdocs build" + task examples_build, "Build the samples": buildSample("directchat") buildSample("helloworld", true) buildSample("circuitrelay", true) - buildTutorial("examples/tutorial_1_connect.md") - buildTutorial("examples/tutorial_2_customproto.md") + buildSample("tutorial_1_connect", true) + buildSample("tutorial_2_customproto", true) # pin system # while nimble lockfile diff --git a/mkdocs.yml b/mkdocs.yml index 9c4fcbb..76c6570 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,9 @@ site_name: nim-libp2p repo_url: https://github.com/status-im/nim-libp2p repo_name: status-im/nim-libp2p site_url: https://status-im.github.io/nim-libp2p/docs -edit_uri: edit/unstable/examples/ +# Can't find a way to point the edit to the .nim instead +# of the .md +edit_uri: '' docs_dir: examples @@ -40,6 +42,7 @@ theme: nav: - Introduction: README.md - Tutorials: - - 'Part I: Simple connection': tutorial_1_connect.md - - 'Part II: Custom protocol': tutorial_2_customproto.md + - 'Simple connection': tutorial_1_connect.md + - 'Create a custom protocol': tutorial_2_customproto.md + - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' diff --git a/tools/markdown_builder.nim b/tools/markdown_builder.nim new file mode 100644 index 0000000..1477c80 --- /dev/null +++ b/tools/markdown_builder.nim @@ -0,0 +1,29 @@ +import os, strutils + +let contents = + if paramCount() > 0: + readFile(paramStr(1)) + else: + stdin.readAll() + +var code = "" +for line in contents.splitLines(true): + let + stripped = line.strip() + isMarkdown = stripped.startsWith("##") + + if isMarkdown: + if code.strip.len > 0: + echo "```nim" + echo code.strip(leading = false) + echo "```" + code = "" + echo(if stripped.len > 3: stripped[3..^1] + else: "") + else: + code &= line +if code.strip.len > 0: + echo "" + echo "```nim" + echo code + echo "```" From bcb8f5e3b6d9349bf0a10c9a5efd2d9a6467b48f Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 29 Sep 2022 10:28:58 +0200 Subject: [PATCH 12/20] Protobuf tutorial (#778) --- examples/tutorial_2_customproto.nim | 1 + examples/tutorial_3_protobuf.nim | 162 ++++++++++++++++++++++++++++ libp2p.nimble | 4 + mkdocs.yml | 1 + 4 files changed, 168 insertions(+) create mode 100644 examples/tutorial_3_protobuf.nim diff --git a/examples/tutorial_2_customproto.nim b/examples/tutorial_2_customproto.nim index 2f5789b..be418a7 100644 --- a/examples/tutorial_2_customproto.nim +++ b/examples/tutorial_2_customproto.nim @@ -71,3 +71,4 @@ proc main() {.async, gcsafe.} = waitFor(main()) ## And that's it! +## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf. diff --git a/examples/tutorial_3_protobuf.nim b/examples/tutorial_3_protobuf.nim new file mode 100644 index 0000000..2af7efe --- /dev/null +++ b/examples/tutorial_3_protobuf.nim @@ -0,0 +1,162 @@ +## # Protobuf usage +## +## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol. +## Most real protocol want their messages to be structured and extensible, which is why +## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to +## define their message structures. +## +## Here, we'll create a slightly more complex protocol, which parses & generate protobuf +## messages. Let's start by importing our dependencies, as usual: +import chronos +import stew/results # for Opt[T] + +import libp2p + +## ## Protobuf encoding & decoding +## This will be the structure of our messages: +## ```protobuf +## message MetricList { +## message Metric { +## string name = 1; +## float value = 2; +## } +## +## repeated Metric metrics = 2; +## } +## ``` +## We'll create our protobuf types, encoders & decoders, according to this format. +## To create the encoders & decoders, we are going to use minprotobuf +## (included in libp2p). +## +## While more modern technics +## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization)) +## exists, minprotobuf is currently the recommended method to handle protobuf, since it has +## been used in production extensively, and audited. +type + Metric = object + name: string + value: float + + MetricList = object + metrics: seq[Metric] + +{.push raises: [].} + +proc encode(m: Metric): ProtoBuffer = + result = initProtoBuffer() + result.write(1, m.name) + result.write(2, m.value) + result.finish() + +proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] = + var res: Metric + let pb = initProtoBuffer(buf) + # "getField" will return a Result[bool, ProtoError]. + # The Result will hold an error if the protobuf is invalid. + # The Result will hold "false" if the field is missing + # + # We are just checking the error, and ignoring whether the value + # is present or not (default values are valid). + discard ? pb.getField(1, res.name) + discard ? pb.getField(2, res.value) + ok(res) + +proc encode(m: MetricList): ProtoBuffer = + result = initProtoBuffer() + for metric in m.metrics: + result.write(1, metric.encode()) + result.finish() + +proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] = + var + res: MetricList + metrics: seq[seq[byte]] + let pb = initProtoBuffer(buf) + discard ? pb.getRepeatedField(1, metrics) + + for metric in metrics: + res.metrics &= ? Metric.decode(metric) + ok(res) + +## ## Results instead of exceptions +## As you can see, this part of the program also uses Results instead of exceptions for error handling. +## We start by `{.push raises: [].}`, which will prevent every non-async function from raising +## exceptions. +## +## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey +## errors to function callers. A `Result[T, E]` will either hold a valid result of type +## T, or an error of type E. +## +## You can check if the call succeeded by using `res.isOk`, and then get the +## value using `res.value` or the error by using `res.error`. +## +## Another useful tool is `?`, which will unpack a Result if it succeeded, +## or if it failed, exit the current procedure returning the error. +## +## nim-result is packed with other functionalities that you'll find in the +## nim-result repository. +## +## Results and exception are generally interchangeable, but have different semantics +## that you may or may not prefer. +## +## ## Creating the protocol +## We'll next create a protocol, like in the last tutorial, to request these metrics from our host +type + MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.} + MetricProto = ref object of LPProtocol + metricGetter: MetricCallback + +proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto = + let res = MetricProto(metricGetter: cb) + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + let + metrics = await res.metricGetter() + asProtobuf = metrics.encode() + await conn.writeLp(asProtobuf.buffer) + await conn.close() + + res.codecs = @["/metric-getter/1.0.0"] + res.handler = handle + return res + +proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} = + let protobuf = await conn.readLp(2048) + # tryGet will raise an exception if the Result contains an error. + # It's useful to bridge between exception-world and result-world + return MetricList.decode(protobuf).tryGet() + +## We can now create our main procedure: +proc main() {.async, gcsafe.} = + let rng = newRng() + proc randomMetricGenerator: Future[MetricList] {.async.} = + let metricCount = rng[].generate(uint32) mod 16 + for i in 0 ..< metricCount + 1: + result.metrics.add(Metric( + name: "metric_" & $i, + value: float(rng[].generate(uint16)) / 1000.0 + )) + return result + let + metricProto1 = MetricProto.new(randomMetricGenerator) + metricProto2 = MetricProto.new(randomMetricGenerator) + switch1 = newStandardSwitch(rng=rng) + switch2 = newStandardSwitch(rng=rng) + + switch1.mount(metricProto1) + + await switch1.start() + await switch2.start() + + let + conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs) + metrics = await metricProto2.fetch(conn) + await conn.close() + + for metric in metrics.metrics: + echo metric.name, " = ", metric.value + + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +waitFor(main()) + +## If you run this program, you should see random metrics being sent from the switch1 to the switch2. diff --git a/libp2p.nimble b/libp2p.nimble index a1c3b04..bc83ef8 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -89,6 +89,7 @@ task test_slim, "Runs the (slimmed down) test suite": task website, "Build the website": tutorialToMd("examples/tutorial_1_connect.nim") tutorialToMd("examples/tutorial_2_customproto.nim") + tutorialToMd("examples/tutorial_3_protobuf.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -98,6 +99,9 @@ task examples_build, "Build the samples": buildSample("circuitrelay", true) buildSample("tutorial_1_connect", true) buildSample("tutorial_2_customproto", true) + if (NimMajor, NimMinor) > (1, 2): + # This tutorial relies on post 1.4 exception tracking + buildSample("tutorial_3_protobuf", true) # pin system # while nimble lockfile diff --git a/mkdocs.yml b/mkdocs.yml index 76c6570..57d8479 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,5 +44,6 @@ nav: - Tutorials: - 'Simple connection': tutorial_1_connect.md - 'Create a custom protocol': tutorial_2_customproto.md + - 'Protobuf': tutorial_3_protobuf.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' From 0cd3554ce471fe53211c881badeabeaec46f4bd8 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 29 Sep 2022 10:29:51 +0200 Subject: [PATCH 13/20] Bump deps (#779) --- .pinned | 18 +++++++++--------- libp2p.nimble | 2 +- libp2p/nameresolving/dnsresolver.nim | 24 ++++++++++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.pinned b/.pinned index f85d7a2..3397fbb 100644 --- a/.pinned +++ b/.pinned @@ -1,16 +1,16 @@ -bearssl;https://github.com/status-im/nim-bearssl@#25009951ff8e0006171d566e3c7dc73a8231c2ed +bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a -chronos;https://github.com/status-im/nim-chronos@#41b82cdea34744148600b67a9154331b76181189 -dnsclient;https://github.com/ba0f3/dnsclient.nim@#4960de2b345f567b12f09a08e9967af104ab39a3 -faststreams;https://github.com/status-im/nim-faststreams@#49e2c52eb5dda46b1c9c10d079abe7bffe6cea89 -httputils;https://github.com/status-im/nim-http-utils@#f83fbce4d6ec7927b75be3f85e4fa905fcb69788 +chronos;https://github.com/status-im/nim-chronos@#9df76c39df254c7ff0cec6dec5c9f345f2819c91 +dnsclient;https://github.com/ba0f3/dnsclient.nim@#6647ca8bd9ffcc13adaecb9cb6453032063967db +faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29 +httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040 json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 -metrics;https://github.com/status-im/nim-metrics@#9070af9c830e93e5239ddc488cd65aa6f609ba73 +metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178 -secp256k1;https://github.com/status-im/nim-secp256k1@#5340cf188168d6afcafc8023770d880f067c0b2f +secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466 -stew;https://github.com/status-im/nim-stew@#598246620da5c41d0e92a8dd6aab0755381b21cd +stew;https://github.com/status-im/nim-stew@#f2e58ba4c8da65548c824e4fa8732db9739f6505 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34 unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db -websock;https://github.com/status-im/nim-websock@#8a72c0f7690802753b1d59887745b1ce1f0c8b3d +websock;https://github.com/status-im/nim-websock@#2424f2b215c0546f97d8b147e21544521c7545b0 zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8 \ No newline at end of file diff --git a/libp2p.nimble b/libp2p.nimble index bc83ef8..eb661d8 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -9,7 +9,7 @@ skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"] requires "nim >= 1.2.0", "nimcrypto >= 0.4.1", - "dnsclient >= 0.1.2", + "dnsclient >= 0.3.0 & < 0.4.0", "bearssl >= 0.1.4", "chronicles >= 0.10.2", "chronos >= 3.0.6", diff --git a/libp2p/nameresolving/dnsresolver.nim b/libp2p/nameresolving/dnsresolver.nim index 51be456..a638b25 100644 --- a/libp2p/nameresolving/dnsresolver.nim +++ b/libp2p/nameresolving/dnsresolver.nim @@ -14,7 +14,7 @@ else: import std/[streams, strutils, sets, sequtils], - chronos, chronicles, + chronos, chronicles, stew/byteutils, dnsclientpkg/[protocol, types] import @@ -76,15 +76,11 @@ proc getDnsResponse( if not receivedDataFuture.finished: raise newException(IOError, "DNS server timeout") - var - rawResponse = sock.getMessage() - dataStream = newStringStream() - dataStream.writeData(addr rawResponse[0], rawResponse.len) - dataStream.setPosition(0) + let rawResponse = sock.getMessage() # parseResponse can has a raises: [Exception, ..] because of # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 # it can't actually raise though - return parseResponse(dataStream) + return parseResponse(string.fromBytes(rawResponse)) except CatchableError as exc: raise exc except Exception as exc: raiseAssert exc.msg finally: @@ -118,7 +114,14 @@ method resolveIp*( try: let resp = await fut for answer in resp.answers: - resolvedAddresses.incl(answer.toString()) + # toString can has a raises: [Exception, ..] because of + # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 + # it can't actually raise though + resolvedAddresses.incl( + try: answer.toString() + except CatchableError as exc: raise exc + except Exception as exc: raiseAssert exc.msg + ) except CancelledError as e: raise e except ValueError as e: @@ -158,6 +161,11 @@ method resolveTxt*( self.nameServers.add(self.nameServers[0]) self.nameServers.delete(0) continue + except Exception as e: + # toString can has a raises: [Exception, ..] because of + # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008 + # it can't actually raise though + raiseAssert e.msg debug "Failed to resolve TXT, returning empty set" return @[] From 4f18dd30e9b9aa9c043cf371765df6a37b8a9265 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Thu, 29 Sep 2022 20:02:10 +0200 Subject: [PATCH 14/20] Handle trying to write empty byte seq (#780) --- libp2p/protocols/connectivity/relay/utils.nim | 14 ++++++++------ libp2p/stream/chronosstream.nim | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libp2p/protocols/connectivity/relay/utils.nim b/libp2p/protocols/connectivity/relay/utils.nim index c5449aa..275a26f 100644 --- a/libp2p/protocols/connectivity/relay/utils.nim +++ b/libp2p/protocols/connectivity/relay/utils.nim @@ -64,15 +64,17 @@ proc bridge*(connSrc: Connection, connDst: Connection) {.async.} = await futSrc or futDst if futSrc.finished(): bufRead = await futSrc - bytesSendFromSrcToDst.inc(bufRead) - await connDst.write(@bufSrcToDst[0.. 0: + bytesSendFromSrcToDst.inc(bufRead) + await connDst.write(@bufSrcToDst[0.. 0: + bytesSendFromDstToSrc += bufRead + await connSrc.write(bufDstToSrc[0.. Date: Fri, 30 Sep 2022 10:41:04 +0200 Subject: [PATCH 15/20] RendezVous Protocol (#751) --- libp2p/builders.nim | 11 +- libp2p/protocols/rendezvous.nim | 679 ++++++++++++++++++++++++++++++++ libp2p/utils/offsettedseq.nim | 73 ++++ tests/testnative.nim | 1 + tests/testrendezvous.nim | 125 ++++++ 5 files changed, 888 insertions(+), 1 deletion(-) create mode 100644 libp2p/protocols/rendezvous.nim create mode 100644 libp2p/utils/offsettedseq.nim create mode 100644 tests/testrendezvous.nim diff --git a/libp2p/builders.nim b/libp2p/builders.nim index 5d111b2..fdff75b 100644 --- a/libp2p/builders.nim +++ b/libp2p/builders.nim @@ -26,7 +26,7 @@ import switch, peerid, peerinfo, stream/connection, multiaddress, crypto/crypto, transports/[transport, tcptransport], muxers/[muxer, mplex/mplex, yamux/yamux], - protocols/[identify, secure/secure, secure/noise], + protocols/[identify, secure/secure, secure/noise, rendezvous], protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport], connmanager, upgrademngrs/muxedupgrade, nameresolving/nameresolver, @@ -60,6 +60,7 @@ type peerStoreCapacity: Option[int] autonat: bool circuitRelay: Relay + rdv: RendezVous proc new*(T: type[SwitchBuilder]): T {.public.} = ## Creates a SwitchBuilder @@ -194,6 +195,10 @@ proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder b.circuitRelay = r b +proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder = + b.rdv = rdv + b + proc build*(b: SwitchBuilder): Switch {.raises: [Defect, LPError], public.} = @@ -261,6 +266,10 @@ proc build*(b: SwitchBuilder): Switch b.circuitRelay.setup(switch) switch.mount(b.circuitRelay) + if not isNil(b.rdv): + b.rdv.setup(switch) + switch.mount(b.rdv) + return switch proc newStandardSwitch*( diff --git a/libp2p/protocols/rendezvous.nim b/libp2p/protocols/rendezvous.nim new file mode 100644 index 0000000..f7d58c9 --- /dev/null +++ b/libp2p/protocols/rendezvous.nim @@ -0,0 +1,679 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import tables, sequtils, sugar, sets, options +import chronos, + chronicles, + bearssl/rand, + stew/[byteutils, objects] +import ./protocol, + ../switch, + ../routing_record, + ../utils/heartbeat, + ../stream/connection, + ../utils/offsettedseq, + ../utils/semaphore + +export chronicles + +logScope: + topics = "libp2p discovery rendezvous" + +const + RendezVousCodec* = "/rendezvous/1.0.0" + MinimumDuration = 2.hours + MaximumDuration = 72.hours + MinimumTTL = MinimumDuration.seconds.uint64 + MaximumTTL = MaximumDuration.seconds.uint64 + RegistrationLimitPerPeer = 1000 + DiscoverLimit = 1000'u64 + SemaphoreDefaultSize = 5 + +type + MessageType {.pure.} = enum + Register = 0 + RegisterResponse = 1 + Unregister = 2 + Discover = 3 + DiscoverResponse = 4 + + ResponseStatus = enum + Ok = 0 + InvalidNamespace = 100 + InvalidSignedPeerRecord = 101 + InvalidTTL = 102 + InvalidCookie = 103 + NotAuthorized = 200 + InternalError = 300 + Unavailable = 400 + + Cookie = object + offset : uint64 + ns : string + + Register = object + ns : string + signedPeerRecord: seq[byte] + ttl: Option[uint64] # in seconds + + RegisterResponse = object + status: ResponseStatus + text: Option[string] + ttl: Option[uint64] # in seconds + + Unregister = object + ns: string + + Discover = object + ns: string + limit: Option[uint64] + cookie: Option[seq[byte]] + + DiscoverResponse = object + registrations: seq[Register] + cookie: Option[seq[byte]] + status: ResponseStatus + text: Option[string] + + Message = object + msgType: MessageType + register: Option[Register] + registerResponse: Option[RegisterResponse] + unregister: Option[Unregister] + discover: Option[Discover] + discoverResponse: Option[DiscoverResponse] + +proc encode(c: Cookie): ProtoBuffer = + result = initProtoBuffer() + result.write(1, c.offset) + result.write(2, c.ns) + result.finish() + +proc encode(r: Register): ProtoBuffer = + result = initProtoBuffer() + result.write(1, r.ns) + result.write(2, r.signedPeerRecord) + if r.ttl.isSome(): + result.write(3, r.ttl.get()) + result.finish() + +proc encode(rr: RegisterResponse): ProtoBuffer = + result = initProtoBuffer() + result.write(1, rr.status.uint) + if rr.text.isSome(): + result.write(2, rr.text.get()) + if rr.ttl.isSome(): + result.write(3, rr.ttl.get()) + result.finish() + +proc encode(u: Unregister): ProtoBuffer = + result = initProtoBuffer() + result.write(1, u.ns) + result.finish() + +proc encode(d: Discover): ProtoBuffer = + result = initProtoBuffer() + result.write(1, d.ns) + if d.limit.isSome(): + result.write(2, d.limit.get()) + if d.cookie.isSome(): + result.write(3, d.cookie.get()) + result.finish() + +proc encode(d: DiscoverResponse): ProtoBuffer = + result = initProtoBuffer() + for reg in d.registrations: + result.write(1, reg.encode()) + if d.cookie.isSome(): + result.write(2, d.cookie.get()) + result.write(3, d.status.uint) + if d.text.isSome(): + result.write(4, d.text.get()) + result.finish() + +proc encode(msg: Message): ProtoBuffer = + result = initProtoBuffer() + result.write(1, msg.msgType.uint) + if msg.register.isSome(): + result.write(2, msg.register.get().encode()) + if msg.registerResponse.isSome(): + result.write(3, msg.registerResponse.get().encode()) + if msg.unregister.isSome(): + result.write(4, msg.unregister.get().encode()) + if msg.discover.isSome(): + result.write(5, msg.discover.get().encode()) + if msg.discoverResponse.isSome(): + result.write(6, msg.discoverResponse.get().encode()) + result.finish() + +proc decode(_: typedesc[Cookie], buf: seq[byte]): Option[Cookie] = + var c: Cookie + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, c.offset) + r2 = pb.getRequiredField(2, c.ns) + if r1.isErr() or r2.isErr(): return none(Cookie) + some(c) + +proc decode(_: typedesc[Register], buf: seq[byte]): Option[Register] = + var + r: Register + ttl: uint64 + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, r.ns) + r2 = pb.getRequiredField(2, r.signedPeerRecord) + r3 = pb.getField(3, ttl) + if r1.isErr() or r2.isErr() or r3.isErr(): return none(Register) + if r3.get(): r.ttl = some(ttl) + some(r) + +proc decode(_: typedesc[RegisterResponse], buf: seq[byte]): Option[RegisterResponse] = + var + rr: RegisterResponse + statusOrd: uint + text: string + ttl: uint64 + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, statusOrd) + r2 = pb.getField(2, text) + r3 = pb.getField(3, ttl) + if r1.isErr() or r2.isErr() or r3.isErr() or + not checkedEnumAssign(rr.status, statusOrd): return none(RegisterResponse) + if r2.get(): rr.text = some(text) + if r3.get(): rr.ttl = some(ttl) + some(rr) + +proc decode(_: typedesc[Unregister], buf: seq[byte]): Option[Unregister] = + var u: Unregister + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, u.ns) + if r1.isErr(): return none(Unregister) + some(u) + +proc decode(_: typedesc[Discover], buf: seq[byte]): Option[Discover] = + var + d: Discover + limit: uint64 + cookie: seq[byte] + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, d.ns) + r2 = pb.getField(2, limit) + r3 = pb.getField(3, cookie) + if r1.isErr() or r2.isErr() or r3.isErr: return none(Discover) + if r2.get(): d.limit = some(limit) + if r3.get(): d.cookie = some(cookie) + some(d) + +proc decode(_: typedesc[DiscoverResponse], buf: seq[byte]): Option[DiscoverResponse] = + var + dr: DiscoverResponse + registrations: seq[seq[byte]] + cookie: seq[byte] + statusOrd: uint + text: string + let + pb = initProtoBuffer(buf) + r1 = pb.getRepeatedField(1, registrations) + r2 = pb.getField(2, cookie) + r3 = pb.getRequiredField(3, statusOrd) + r4 = pb.getField(4, text) + if r1.isErr() or r2.isErr() or r3.isErr or r4.isErr() or + not checkedEnumAssign(dr.status, statusOrd): return none(DiscoverResponse) + for reg in registrations: + var r: Register + let regOpt = Register.decode(reg) + if regOpt.isNone(): return none(DiscoverResponse) + dr.registrations.add(regOpt.get()) + if r2.get(): dr.cookie = some(cookie) + if r4.get(): dr.text = some(text) + some(dr) + +proc decode(_: typedesc[Message], buf: seq[byte]): Option[Message] = + var + msg: Message + statusOrd: uint + pbr, pbrr, pbu, pbd, pbdr: ProtoBuffer + let + pb = initProtoBuffer(buf) + r1 = pb.getRequiredField(1, statusOrd) + r2 = pb.getField(2, pbr) + r3 = pb.getField(3, pbrr) + r4 = pb.getField(4, pbu) + r5 = pb.getField(5, pbd) + r6 = pb.getField(6, pbdr) + if r1.isErr() or r2.isErr() or r3.isErr() or + r4.isErr() or r5.isErr() or r6.isErr() or + not checkedEnumAssign(msg.msgType, statusOrd): return none(Message) + if r2.get(): + msg.register = Register.decode(pbr.buffer) + if msg.register.isNone(): return none(Message) + if r3.get(): + msg.registerResponse = RegisterResponse.decode(pbrr.buffer) + if msg.registerResponse.isNone(): return none(Message) + if r4.get(): + msg.unregister = Unregister.decode(pbu.buffer) + if msg.unregister.isNone(): return none(Message) + if r5.get(): + msg.discover = Discover.decode(pbd.buffer) + if msg.discover.isNone(): return none(Message) + if r6.get(): + msg.discoverResponse = DiscoverResponse.decode(pbdr.buffer) + if msg.discoverResponse.isNone(): return none(Message) + some(msg) + + +type + RendezVousError* = object of LPError + RegisteredData = object + expiration: Moment + peerId: PeerId + data: Register + + RegisteredSeq = object + s: seq[RegisteredData] + offset: uint64 + + RendezVous* = ref object of LPProtocol + # Registered needs to be an offsetted sequence + # because we need stable index for the cookies. + registered: OffsettedSeq[RegisteredData] + # Namespaces is a table whose key is a salted namespace and + # the value is the index sequence corresponding to this + # namespace in the offsettedqueue. + namespaces: Table[string, seq[int]] + rng: ref HmacDrbgContext + salt: string + defaultDT: Moment + registerDeletionLoop: Future[void] + #registerEvent: AsyncEvent # TODO: to raise during the heartbeat + # + make the heartbeat sleep duration "smarter" + sema: AsyncSemaphore + peers: seq[PeerId] + cookiesSaved: Table[PeerId, Table[string, seq[byte]]] + switch: Switch + +proc checkPeerRecord(spr: seq[byte], peerId: PeerId): Result[void, string] = + if spr.len == 0: return err("Empty peer record") + let signedEnv = ? SignedPeerRecord.decode(spr).mapErr(x => $x) + if signedEnv.data.peerId != peerId: + return err("Bad Peer ID") + return ok() + +proc sendRegisterResponse(conn: Connection, + ttl: uint64) {.async.} = + let msg = encode(Message( + msgType: MessageType.RegisterResponse, + registerResponse: some(RegisterResponse(status: Ok, ttl: some(ttl))))) + await conn.writeLp(msg.buffer) + +proc sendRegisterResponseError(conn: Connection, + status: ResponseStatus, + text: string = "") {.async.} = + let msg = encode(Message( + msgType: MessageType.RegisterResponse, + registerResponse: some(RegisterResponse(status: status, text: some(text))))) + await conn.writeLp(msg.buffer) + +proc sendDiscoverResponse(conn: Connection, + s: seq[Register], + cookie: Cookie) {.async.} = + let msg = encode(Message( + msgType: MessageType.DiscoverResponse, + discoverResponse: some(DiscoverResponse( + status: Ok, + registrations: s, + cookie: some(cookie.encode().buffer) + )) + )) + await conn.writeLp(msg.buffer) + +proc sendDiscoverResponseError(conn: Connection, + status: ResponseStatus, + text: string = "") {.async.} = + let msg = encode(Message( + msgType: MessageType.DiscoverResponse, + discoverResponse: some(DiscoverResponse(status: status, text: some(text))))) + await conn.writeLp(msg.buffer) + +proc countRegister(rdv: RendezVous, peerId: PeerId): int = + let n = Moment.now() + for data in rdv.registered: + if data.peerId == peerId and data.expiration > n: + result.inc() + +proc save(rdv: RendezVous, + ns: string, + peerId: PeerId, + r: Register, + update: bool = true) = + let nsSalted = ns & rdv.salt + discard rdv.namespaces.hasKeyOrPut(nsSalted, newSeq[int]()) + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == peerId: + if update == false: return + rdv.registered[index].expiration = rdv.defaultDT + rdv.registered.add( + RegisteredData( + peerId: peerId, + expiration: Moment.now() + r.ttl.get(MinimumTTL).int64.seconds, + data: r + ) + ) + rdv.namespaces[nsSalted].add(rdv.registered.high) +# rdv.registerEvent.fire() + except KeyError: + doAssert false, "Should have key" + +proc register(rdv: RendezVous, conn: Connection, r: Register): Future[void] = + trace "Received Register", peerId = conn.peerId, ns = r.ns + if r.ns.len notin 1..255: + return conn.sendRegisterResponseError(InvalidNamespace) + let ttl = r.ttl.get(MinimumTTL) + if ttl notin MinimumTTL..MaximumTTL: + return conn.sendRegisterResponseError(InvalidTTL) + let pr = checkPeerRecord(r.signedPeerRecord, conn.peerId) + if pr.isErr(): + return conn.sendRegisterResponseError(InvalidSignedPeerRecord, pr.error()) + if rdv.countRegister(conn.peerId) >= RegistrationLimitPerPeer: + return conn.sendRegisterResponseError(NotAuthorized, "Registration limit reached") + rdv.save(r.ns, conn.peerId, r) + conn.sendRegisterResponse(ttl) + +proc unregister(rdv: RendezVous, conn: Connection, u: Unregister) = + trace "Received Unregister", peerId = conn.peerId, ns = u.ns + let nsSalted = u.ns & rdv.salt + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == conn.peerId: + rdv.registered[index].expiration = rdv.defaultDT + except KeyError: + return + +proc discover(rdv: RendezVous, conn: Connection, d: Discover) {.async.} = + trace "Received Discover", peerId = conn.peerId, ns = d.ns + if d.ns.len notin 0..255: + await conn.sendDiscoverResponseError(InvalidNamespace) + return + var limit = min(DiscoverLimit, d.limit.get(DiscoverLimit)) + var + cookie = + if d.cookie.isSome(): + try: + Cookie.decode(d.cookie.get()).get() + except CatchableError: + await conn.sendDiscoverResponseError(InvalidCookie) + return + else: Cookie(offset: rdv.registered.low().uint64 - 1) + if cookie.ns != d.ns or + cookie.offset notin rdv.registered.low().uint64..rdv.registered.high().uint64: + cookie = Cookie(offset: rdv.registered.low().uint64 - 1) + let + nsSalted = d.ns & rdv.salt + namespaces = + if d.ns != "": + try: + rdv.namespaces[nsSalted] + except KeyError: + await conn.sendDiscoverResponseError(InvalidNamespace) + return + else: toSeq(cookie.offset.int..rdv.registered.high()) + if namespaces.len() == 0: + await conn.sendDiscoverResponse(@[], Cookie()) + return + var offset = namespaces[^1] + let n = Moment.now() + var s = collect(newSeq()): + for index in namespaces: + var reg = rdv.registered[index] + if limit == 0: + offset = index + break + if reg.expiration < n or index.uint64 <= cookie.offset: continue + limit.dec() + reg.data.ttl = some((reg.expiration - Moment.now()).seconds.uint64) + reg.data + rdv.rng.shuffle(s) + await conn.sendDiscoverResponse(s, Cookie(offset: offset.uint64, ns: d.ns)) + +proc advertisePeer(rdv: RendezVous, + peer: PeerId, + msg: seq[byte]) {.async.} = + proc advertiseWrap() {.async.} = + try: + let conn = await rdv.switch.dial(peer, RendezVousCodec) + defer: await conn.close() + await conn.writeLp(msg) + let + buf = await conn.readLp(4096) + msgRecv = Message.decode(buf).get() + if msgRecv.msgType != MessageType.RegisterResponse: + trace "Unexpected register response", peer, msgType = msgRecv.msgType + elif msgRecv.registerResponse.isNone() or + msgRecv.registerResponse.get().status != ResponseStatus.Ok: + trace "Refuse to register", peer, response = msgRecv.registerResponse + except CatchableError as exc: + trace "exception in the advertise", error = exc.msg + finally: + rdv.sema.release() + await rdv.sema.acquire() + discard await advertiseWrap().withTimeout(5.seconds) + +proc advertise*(rdv: RendezVous, + ns: string, + ttl: Duration = MinimumDuration) {.async.} = + let sprBuff = rdv.switch.peerInfo.signedPeerRecord.encode() + if sprBuff.isErr(): + raise newException(RendezVousError, "Wrong Signed Peer Record") + if ns.len notin 1..255: + raise newException(RendezVousError, "Invalid namespace") + if ttl notin MinimumDuration..MaximumDuration: + raise newException(RendezVousError, "Invalid time to live") + let + r = Register(ns: ns, signedPeerRecord: sprBuff.get(), ttl: some(ttl.seconds.uint64)) + msg = encode(Message(msgType: MessageType.Register, register: some(r))) + rdv.save(ns, rdv.switch.peerInfo.peerId, r) + let fut = collect(newSeq()): + for peer in rdv.peers: + trace "Send Advertise", peerId = peer, ns + rdv.advertisePeer(peer, msg.buffer) + await allFutures(fut) + +proc requestLocally*(rdv: RendezVous, ns: string): seq[PeerRecord] = + let + nsSalted = ns & rdv.salt + n = Moment.now() + try: + collect(newSeq()): + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].expiration > n: + SignedPeerRecord.decode(rdv.registered[index].data.signedPeerRecord).get().data + except KeyError as exc: + @[] + +proc request*(rdv: RendezVous, + ns: string, + l: int = DiscoverLimit.int): Future[seq[PeerRecord]] {.async.} = + let nsSalted = ns & rdv.salt + var + s: Table[PeerId, (PeerRecord, Register)] + limit: uint64 + d = Discover(ns: ns) + + if l <= 0 or l > DiscoverLimit.int: + raise newException(RendezVousError, "Invalid limit") + if ns.len notin 0..255: + raise newException(RendezVousError, "Invalid namespace") + limit = l.uint64 + proc requestPeer(peer: PeerId) {.async.} = + let conn = await rdv.switch.dial(peer, RendezVousCodec) + defer: await conn.close() + d.limit = some(limit) + d.cookie = + try: + some(rdv.cookiesSaved[peer][ns]) + except KeyError as exc: + none(seq[byte]) + await conn.writeLp(encode(Message( + msgType: MessageType.Discover, + discover: some(d))).buffer) + let + buf = await conn.readLp(65536) + msgRcv = Message.decode(buf).get() + if msgRcv.msgType != MessageType.DiscoverResponse or + msgRcv.discoverResponse.isNone(): + debug "Unexpected discover response", msgType = msgRcv.msgType + return + let resp = msgRcv.discoverResponse.get() + if resp.status != ResponseStatus.Ok: + trace "Cannot discover", ns, status = resp.status, text = resp.text + return + if resp.cookie.isSome() and resp.cookie.get().len < 1000: + if rdv.cookiesSaved.hasKeyOrPut(peer, {ns: resp.cookie.get()}.toTable): + rdv.cookiesSaved[peer][ns] = resp.cookie.get() + for r in resp.registrations: + if limit == 0: return + if r.ttl.isNone() or r.ttl.get() > MaximumTTL: continue + let sprRes = SignedPeerRecord.decode(r.signedPeerRecord) + if sprRes.isErr(): continue + let pr = sprRes.get().data + if s.hasKey(pr.peerId): + let (prSaved, rSaved) = s[pr.peerId] + if (prSaved.seqNo == pr.seqNo and rSaved.ttl.get() < r.ttl.get()) or + prSaved.seqNo < pr.seqNo: + s[pr.peerId] = (pr, r) + else: + s[pr.peerId] = (pr, r) + limit.dec() + for (_, r) in s.values(): + rdv.save(ns, peer, r, false) + + for peer in rdv.peers: + if limit == 0: break + if RendezVousCodec notin rdv.switch.peerStore[ProtoBook][peer]: continue + try: + trace "Send Request", peerId = peer, ns + await peer.requestPeer() + except CancelledError as exc: + raise exc + except CatchableError as exc: + trace "exception catch in request", error = exc.msg + return toSeq(s.values()).mapIt(it[0]) + +proc unsubscribeLocally*(rdv: RendezVous, ns: string) = + let nsSalted = ns & rdv.salt + try: + for index in rdv.namespaces[nsSalted]: + if rdv.registered[index].peerId == rdv.switch.peerInfo.peerId: + rdv.registered[index].expiration = rdv.defaultDT + except KeyError: + return + +proc unsubscribe*(rdv: RendezVous, ns: string) {.async.} = + # TODO: find a way to improve this, maybe something similar to the advertise + if ns.len notin 1..255: + raise newException(RendezVousError, "Invalid namespace") + rdv.unsubscribeLocally(ns) + let msg = encode(Message( + msgType: MessageType.Unregister, + unregister: some(Unregister(ns: ns)))) + + proc unsubscribePeer(rdv: RendezVous, peerId: PeerId) {.async.} = + try: + let conn = await rdv.switch.dial(peerId, RendezVousCodec) + defer: await conn.close() + await conn.writeLp(msg.buffer) + except CatchableError as exc: + trace "exception while unsubscribing", error = exc.msg + + for peer in rdv.peers: + discard await rdv.unsubscribePeer(peer).withTimeout(5.seconds) + +proc setup*(rdv: RendezVous, switch: Switch) = + rdv.switch = switch + proc handlePeer(peerId: PeerId, event: PeerEvent) {.async.} = + if event.kind == PeerEventKind.Joined: + rdv.peers.add(peerId) + elif event.kind == PeerEventKind.Left: + rdv.peers.keepItIf(it != peerId) + rdv.switch.addPeerEventHandler(handlePeer, Joined) + rdv.switch.addPeerEventHandler(handlePeer, Left) + +proc new*(T: typedesc[RendezVous], + rng: ref HmacDrbgContext = newRng()): T = + let rdv = T( + rng: rng, + salt: string.fromBytes(generateBytes(rng[], 8)), + registered: initOffsettedSeq[RegisteredData](1), + defaultDT: Moment.now() - 1.days, + #registerEvent: newAsyncEvent(), + sema: newAsyncSemaphore(SemaphoreDefaultSize) + ) + logScope: topics = "libp2p discovery rendezvous" + proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} = + try: + let + buf = await conn.readLp(4096) + msg = Message.decode(buf).get() + case msg.msgType: + of MessageType.Register: await rdv.register(conn, msg.register.get()) + of MessageType.RegisterResponse: + trace "Got an unexpected Register Response", response = msg.registerResponse + of MessageType.Unregister: rdv.unregister(conn, msg.unregister.get()) + of MessageType.Discover: await rdv.discover(conn, msg.discover.get()) + of MessageType.DiscoverResponse: + trace "Got an unexpected Discover Response", response = msg.discoverResponse + except CancelledError as exc: + raise exc + except CatchableError as exc: + trace "exception in rendezvous handler", error = exc.msg + finally: + await conn.close() + + rdv.handler = handleStream + rdv.codec = RendezVousCodec + return rdv + +proc new*(T: typedesc[RendezVous], + switch: Switch, + rng: ref HmacDrbgContext = newRng()): T = + let rdv = T.new(rng) + rdv.setup(switch) + return rdv + +proc deletesRegister(rdv: RendezVous) {.async.} = + heartbeat "Register timeout", 1.minutes: + let n = Moment.now() + rdv.registered.flushIfIt(it.expiration < n) + for data in rdv.namespaces.mvalues(): + data.keepItIf(it >= rdv.registered.offset) + +method start*(rdv: RendezVous) {.async.} = + if not rdv.registerDeletionLoop.isNil: + warn "Starting rendezvous twice" + return + rdv.registerDeletionLoop = rdv.deletesRegister() + rdv.started = true + +method stop*(rdv: RendezVous) {.async.} = + if rdv.registerDeletionLoop.isNil: + warn "Stopping rendezvous without starting it" + return + rdv.started = false + rdv.registerDeletionLoop.cancel() + rdv.registerDeletionLoop = nil diff --git a/libp2p/utils/offsettedseq.nim b/libp2p/utils/offsettedseq.nim new file mode 100644 index 0000000..4459531 --- /dev/null +++ b/libp2p/utils/offsettedseq.nim @@ -0,0 +1,73 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import sequtils + +type + OffsettedSeq*[T] = object + s*: seq[T] + offset*: int + +proc initOffsettedSeq*[T](offset: int = 0): OffsettedSeq[T] = + OffsettedSeq[T](s: newSeq[T](), offset: offset) + +proc all*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool = + o.s.all(pred) + +proc any*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool = + o.s.any(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: T)) = + o.s.apply(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: T): T) = + o.s.apply(pred) + +proc apply*[T](o: OffsettedSeq[T], op: proc (x: var T)) = + o.s.apply(pred) + +func count*[T](o: OffsettedSeq[T], x: T): int = + o.s.count(x) + +proc flushIf*[T](o: OffsettedSeq[T], pred: proc (x: T): bool) = + var i = 0 + for e in o.s: + if not pred(e): break + i.inc() + if i > 0: + o.s.delete(0.. 0: + when (NimMajor, NimMinor) < (1, 4): + o.s.delete(0, i - 1) + else: + o.s.delete(0.. Date: Tue, 4 Oct 2022 00:00:00 +0200 Subject: [PATCH 16/20] Discovery interface (#783) Co-authored-by: Tanguy --- libp2p/discovery/discoverymngr.nim | 163 +++++++++++++++++++++++ libp2p/discovery/rendezvousinterface.nim | 77 +++++++++++ libp2p/protocols/rendezvous.nim | 6 +- tests/testdiscovery.nim | 51 +++++++ tests/testnative.nim | 1 + 5 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 libp2p/discovery/discoverymngr.nim create mode 100644 libp2p/discovery/rendezvousinterface.nim create mode 100644 tests/testdiscovery.nim diff --git a/libp2p/discovery/discoverymngr.nim b/libp2p/discovery/discoverymngr.nim new file mode 100644 index 0000000..6c18543 --- /dev/null +++ b/libp2p/discovery/discoverymngr.nim @@ -0,0 +1,163 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/sequtils +import chronos, chronicles, stew/results +import ../errors + +type + BaseAttr = ref object of RootObj + comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [Defect].} + + Attribute[T] = ref object of BaseAttr + value: T + + PeerAttributes* = object + attributes: seq[BaseAttr] + + DiscoveryService* = distinct string + +proc `==`*(a, b: DiscoveryService): bool {.borrow.} + +proc ofType*[T](f: BaseAttr, _: type[T]): bool = + return f of Attribute[T] + +proc to*[T](f: BaseAttr, _: type[T]): T = + Attribute[T](f).value + +proc add*[T](pa: var PeerAttributes, + value: T) = + pa.attributes.add(Attribute[T]( + value: value, + comparator: proc(f: BaseAttr, c: BaseAttr): bool = + f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T) + ) + ) + +iterator items*(pa: PeerAttributes): BaseAttr = + for f in pa.attributes: + yield f + +proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] = + for f in pa.attributes: + if f.ofType(T): + result.add(f.to(T)) + +proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] = + for f in pa.attributes: + if f.ofType(T): + return Opt.some(f.to(T)) + Opt.none(T) + +proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [Defect, KeyError].} = + pa{T}.valueOr: raise newException(KeyError, "Attritute not found") + +proc match*(pa, candidate: PeerAttributes): bool = + for f in pa.attributes: + block oneAttribute: + for field in candidate.attributes: + if field.comparator(field, f): + break oneAttribute + return false + return true + +type + PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [Defect], gcsafe.} + + DiscoveryInterface* = ref object of RootObj + onPeerFound*: PeerFoundCallback + toAdvertise*: PeerAttributes + advertisementUpdated*: AsyncEvent + advertiseLoop*: Future[void] + +method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} = + doAssert(false, "Not implemented!") + +method advertise*(self: DiscoveryInterface) {.async, base.} = + doAssert(false, "Not implemented!") + +type + DiscoveryError* = object of LPError + + DiscoveryQuery* = ref object + attr: PeerAttributes + peers: AsyncQueue[PeerAttributes] + futs: seq[Future[void]] + + DiscoveryManager* = ref object + interfaces: seq[DiscoveryInterface] + queries: seq[DiscoveryQuery] + +proc add*(dm: DiscoveryManager, di: DiscoveryInterface) = + dm.interfaces &= di + + di.onPeerFound = proc (pa: PeerAttributes) = + for query in dm.queries: + if query.attr.match(pa): + try: + query.peers.putNoWait(pa) + except AsyncQueueFullError as exc: + debug "Cannot push discovered peer to queue" + +proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery = + var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]()) + for i in dm.interfaces: + query.futs.add(i.request(pa)) + dm.queries.add(query) + dm.queries.keepItIf(it.futs.anyIt(not it.finished())) + return query + +proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery = + var pa: PeerAttributes + pa.add(value) + return dm.request(pa) + +proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) = + for i in dm.interfaces: + i.toAdvertise = pa + if i.advertiseLoop.isNil: + i.advertisementUpdated = newAsyncEvent() + i.advertiseLoop = i.advertise() + else: + i.advertisementUpdated.fire() + +proc advertise*[T](dm: DiscoveryManager, value: T) = + var pa: PeerAttributes + pa.add(value) + dm.advertise(pa) + +proc stop*(query: DiscoveryQuery) = + for r in query.futs: + if not r.finished(): r.cancel() + +proc stop*(dm: DiscoveryManager) = + for q in dm.queries: + q.stop() + for i in dm.interfaces: + if isNil(i.advertiseLoop): continue + i.advertiseLoop.cancel() + +proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} = + let getter = query.peers.popFirst() + + try: + await getter or allFinished(query.futs) + except CancelledError as exc: + getter.cancel() + raise exc + + if not finished(getter): + # discovery loops only finish when they don't handle the query + raise newException(DiscoveryError, "Unable to find any peer matching this request") + return await getter diff --git a/libp2p/discovery/rendezvousinterface.nim b/libp2p/discovery/rendezvousinterface.nim new file mode 100644 index 0000000..d3196e8 --- /dev/null +++ b/libp2p/discovery/rendezvousinterface.nim @@ -0,0 +1,77 @@ +# Nim-LibP2P +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import sequtils +import chronos +import ./discoverymngr, + ../protocols/rendezvous, + ../peerid + +type + RendezVousInterface* = ref object of DiscoveryInterface + rdv*: RendezVous + timeToRequest: Duration + timeToAdvertise: Duration + + RdvNamespace* = distinct string + +proc `==`*(a, b: RdvNamespace): bool {.borrow.} + +method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} = + var namespace = "" + for attr in pa: + if attr.ofType(RdvNamespace): + namespace = string attr.to(RdvNamespace) + elif attr.ofType(DiscoveryService): + namespace = string attr.to(DiscoveryService) + elif attr.ofType(PeerId): + namespace = $attr.to(PeerId) + else: + # unhandled type + return + while true: + for pr in await self.rdv.request(namespace): + var peer: PeerAttributes + peer.add(pr.peerId) + for address in pr.addresses: + peer.add(address.address) + + peer.add(DiscoveryService(namespace)) + peer.add(RdvNamespace(namespace)) + self.onPeerFound(peer) + + await sleepAsync(self.timeToRequest) + +method advertise*(self: RendezVousInterface) {.async.} = + while true: + var toAdvertise: seq[string] + for attr in self.toAdvertise: + if attr.ofType(RdvNamespace): + toAdvertise.add string attr.to(RdvNamespace) + elif attr.ofType(DiscoveryService): + toAdvertise.add string attr.to(DiscoveryService) + elif attr.ofType(PeerId): + toAdvertise.add $attr.to(PeerId) + + self.advertisementUpdated.clear() + for toAdv in toAdvertise: + await self.rdv.advertise(toAdv, self.timeToAdvertise) + + await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait() + +proc new*(T: typedesc[RendezVousInterface], + rdv: RendezVous, + ttr: Duration = 1.minutes, + tta: Duration = MinimumDuration): RendezVousInterface = + T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta) diff --git a/libp2p/protocols/rendezvous.nim b/libp2p/protocols/rendezvous.nim index f7d58c9..5460cdc 100644 --- a/libp2p/protocols/rendezvous.nim +++ b/libp2p/protocols/rendezvous.nim @@ -32,7 +32,7 @@ logScope: const RendezVousCodec* = "/rendezvous/1.0.0" - MinimumDuration = 2.hours + MinimumDuration* = 2.hours MaximumDuration = 72.hours MinimumTTL = MinimumDuration.seconds.uint64 MaximumTTL = MaximumDuration.seconds.uint64 @@ -284,10 +284,6 @@ type peerId: PeerId data: Register - RegisteredSeq = object - s: seq[RegisteredData] - offset: uint64 - RendezVous* = ref object of LPProtocol # Registered needs to be an offsetted sequence # because we need stable index for the cookies. diff --git a/tests/testdiscovery.nim b/tests/testdiscovery.nim new file mode 100644 index 0000000..58ebf93 --- /dev/null +++ b/tests/testdiscovery.nim @@ -0,0 +1,51 @@ +{.used.} + +import options, chronos, sets +import stew/byteutils +import ../libp2p/[protocols/rendezvous, + switch, + builders, + discovery/discoverymngr, + discovery/rendezvousinterface,] +import ./helpers + +proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch = + SwitchBuilder.new() + .withRng(newRng()) + .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ]) + .withTcpTransport() + .withMplex() + .withNoise() + .withRendezVous(rdv) + .build() + +suite "Discovery": + teardown: + checkTrackers() + asyncTest "RendezVous test": + let + rdvA = RendezVous.new() + rdvB = RendezVous.new() + clientA = createSwitch(rdvA) + clientB = createSwitch(rdvB) + remoteNode = createSwitch() + dmA = DiscoveryManager() + dmB = DiscoveryManager() + dmA.add(RendezVousInterface.new(rdvA, ttr = 500.milliseconds)) + dmB.add(RendezVousInterface.new(rdvB)) + await allFutures(clientA.start(), clientB.start(), remoteNode.start()) + + await clientB.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs) + await clientA.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs) + + dmB.advertise(RdvNamespace("foo")) + + let + query = dmA.request(RdvNamespace("foo")) + res = await query.getPeer() + check: + res{PeerId}.get() == clientB.peerInfo.peerId + res[PeerId] == clientB.peerInfo.peerId + res.getAll(PeerId) == @[clientB.peerInfo.peerId] + toHashSet(res.getAll(MultiAddress)) == toHashSet(clientB.peerInfo.addrs) + await allFutures(clientA.stop(), clientB.stop(), remoteNode.stop()) diff --git a/tests/testnative.nim b/tests/testnative.nim index 3493c0f..f35971b 100644 --- a/tests/testnative.nim +++ b/tests/testnative.nim @@ -38,5 +38,6 @@ import testtcptransport, testrelayv1, testrelayv2, testrendezvous, + testdiscovery, testyamux, testautonat From 7b103e02f2fe93ad9e367a9620e5117a18437eb2 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 20 Oct 2022 12:22:28 +0200 Subject: [PATCH 17/20] Allow public address mapping (#767) --- examples/circuitrelay.nim | 14 +++++------ libp2p/peerinfo.nim | 35 ++++++++++++++++++++-------- libp2p/switch.nim | 8 +++---- tests/testidentify.nim | 14 +++++------ tests/testnoise.nim | 3 +-- tests/testpeerinfo.nim | 24 +++++++++++++++---- tests/testrelayv2.nim | 49 ++++++++++++++++++++++++++------------- 7 files changed, 97 insertions(+), 50 deletions(-) diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index bf90d97..accbd11 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -48,19 +48,19 @@ proc main() {.async.} = swSrc = createCircuitRelaySwitch(clSrc) swDst = createCircuitRelaySwitch(clDst) - # Create a relay address to swDst using swRel as the relay - addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" & - $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $swDst.peerInfo.peerId).get() - swDst.mount(proto) await swRel.start() await swSrc.start() await swDst.start() - # Connect both Src and Dst to the relay, but not to each other. - await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs) + let + # Create a relay address to swDst using swRel as the relay + addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" & + $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $swDst.peerInfo.peerId).get() + + # Connect Dst to the relay await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs) # Dst reserve a slot on the relay. diff --git a/libp2p/peerinfo.nim b/libp2p/peerinfo.nim index 59fbe74..39a6ad6 100644 --- a/libp2p/peerinfo.nim +++ b/libp2p/peerinfo.nim @@ -22,11 +22,17 @@ export peerid, multiaddress, crypto, routing_record, errors, results ## Our local peer info type - PeerInfoError* = LPError + PeerInfoError* = object of LPError + + AddressMapper* = + proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] + {.gcsafe, raises: [Defect].} PeerInfo* {.public.} = ref object peerId*: PeerId - addrs*: seq[MultiAddress] + listenAddrs*: seq[MultiAddress] + addrs: seq[MultiAddress] + addressMappers*: seq[AddressMapper] protocols*: seq[string] protoVersion*: string agentVersion*: string @@ -37,6 +43,7 @@ type func shortLog*(p: PeerInfo): auto = ( peerId: $p.peerId, + listenAddrs: mapIt(p.listenAddrs, $it), addrs: mapIt(p.addrs, $it), protocols: mapIt(p.protocols, $it), protoVersion: p.protoVersion, @@ -44,7 +51,11 @@ func shortLog*(p: PeerInfo): auto = ) chronicles.formatIt(PeerInfo): shortLog(it) -proc update*(p: PeerInfo) = +proc update*(p: PeerInfo) {.async.} = + p.addrs = p.listenAddrs + for mapper in p.addressMappers: + p.addrs = await mapper(p.addrs) + let sprRes = SignedPeerRecord.init( p.privateKey, PeerRecord.init(p.peerId, p.addrs) @@ -55,20 +66,25 @@ proc update*(p: PeerInfo) = discard #info "Can't update the signed peer record" +proc addrs*(p: PeerInfo): seq[MultiAddress] = + p.addrs + proc new*( p: typedesc[PeerInfo], key: PrivateKey, - addrs: openArray[MultiAddress] = [], + listenAddrs: openArray[MultiAddress] = [], protocols: openArray[string] = [], protoVersion: string = "", - agentVersion: string = ""): PeerInfo - {.raises: [Defect, PeerInfoError].} = + agentVersion: string = "", + addressMappers = newSeq[AddressMapper](), + ): PeerInfo + {.raises: [Defect, LPError].} = let pubkey = try: key.getPublicKey().tryGet() except CatchableError: raise newException(PeerInfoError, "invalid private key") - + let peerId = PeerId.init(key).tryGet() let peerInfo = PeerInfo( @@ -77,10 +93,9 @@ proc new*( privateKey: key, protoVersion: protoVersion, agentVersion: agentVersion, - addrs: @addrs, + listenAddrs: @listenAddrs, protocols: @protocols, + addressMappers: addressMappers ) - peerInfo.update() - return peerInfo diff --git a/libp2p/switch.nim b/libp2p/switch.nim index 93f2b5f..b6fc22f 100644 --- a/libp2p/switch.nim +++ b/libp2p/switch.nim @@ -299,11 +299,11 @@ proc start*(s: Switch) {.async, gcsafe, public.} = trace "starting switch for peer", peerInfo = s.peerInfo var startFuts: seq[Future[void]] for t in s.transports: - let addrs = s.peerInfo.addrs.filterIt( + let addrs = s.peerInfo.listenAddrs.filterIt( t.handles(it) ) - s.peerInfo.addrs.keepItIf( + s.peerInfo.listenAddrs.keepItIf( it notin addrs ) @@ -320,9 +320,9 @@ proc start*(s: Switch) {.async, gcsafe, public.} = for t in s.transports: # for each transport if t.addrs.len > 0 or t.running: s.acceptFuts.add(s.accept(t)) - s.peerInfo.addrs &= t.addrs + s.peerInfo.listenAddrs &= t.addrs - s.peerInfo.update() + await s.peerInfo.update() await s.ms.start() diff --git a/tests/testidentify.nim b/tests/testidentify.nim index 8fc5292..b83eecd 100644 --- a/tests/testidentify.nim +++ b/tests/testidentify.nim @@ -52,6 +52,9 @@ suite "Identify": msListen = MultistreamSelect.new() msDial = MultistreamSelect.new() + serverFut = transport1.start(ma) + await remotePeerInfo.update() + asyncTeardown: await conn.close() await acceptFut @@ -61,7 +64,6 @@ suite "Identify": asyncTest "default agent version": msListen.addHandler(IdentifyCodec, identifyProto1) - serverFut = transport1.start(ma) proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -84,8 +86,6 @@ suite "Identify": remotePeerInfo.agentVersion = customAgentVersion msListen.addHandler(IdentifyCodec, identifyProto1) - serverFut = transport1.start(ma) - proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -105,7 +105,6 @@ suite "Identify": asyncTest "handle failed identify": msListen.addHandler(IdentifyCodec, identifyProto1) - asyncSpawn transport1.start(ma) proc acceptHandler() {.async.} = var conn: Connection @@ -128,7 +127,6 @@ suite "Identify": asyncTest "can send signed peer record": msListen.addHandler(IdentifyCodec, identifyProto1) identifyProto1.sendSignedPeerRecord = true - serverFut = transport1.start(ma) proc acceptHandler(): Future[void] {.async, gcsafe.} = let c = await transport1.accept() await msListen.handle(c) @@ -195,7 +193,8 @@ suite "Identify": asyncTest "simple push identify": switch2.peerInfo.protocols.add("/newprotocol/") - switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + await switch2.peerInfo.update() check: switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs @@ -216,7 +215,8 @@ suite "Identify": asyncTest "wrong peer id push identify": switch2.peerInfo.protocols.add("/newprotocol/") - switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet()) + await switch2.peerInfo.update() check: switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs diff --git a/tests/testnoise.nim b/tests/testnoise.nim index a94714c..f715a7b 100644 --- a/tests/testnoise.nim +++ b/tests/testnoise.nim @@ -60,8 +60,7 @@ method init(p: TestProto) {.gcsafe.} = proc createSwitch(ma: MultiAddress; outgoing: bool, secio: bool = false): (Switch, PeerInfo) = var privateKey = PrivateKey.random(ECDSA, rng[]).get() - peerInfo = PeerInfo.new(privateKey) - peerInfo.addrs.add(ma) + peerInfo = PeerInfo.new(privateKey, @[ma]) proc createMplex(conn: Connection): Muxer = result = Mplex.new(conn) diff --git a/tests/testpeerinfo.nim b/tests/testpeerinfo.nim index 339836b..e4f2739 100644 --- a/tests/testpeerinfo.nim +++ b/tests/testpeerinfo.nim @@ -18,22 +18,24 @@ suite "PeerInfo": check peerId == peerInfo.peerId check seckey.getPublicKey().get() == peerInfo.publicKey - + test "Signed peer record": const ExpectedDomain = $multiCodec("libp2p-peer-record") ExpectedPayloadType = @[(byte) 0x03, (byte) 0x01] - + let seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey).get() multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()] peerInfo = PeerInfo.new(seckey, multiAddresses) - + + waitFor(peerInfo.update()) + let env = peerInfo.signedPeerRecord.envelope rec = PeerRecord.decode(env.payload()).tryGet() - + # Check envelope fields check: env.publicKey == peerInfo.publicKey @@ -47,3 +49,17 @@ suite "PeerInfo": rec.addresses.len == 2 rec.addresses[0].address == multiAddresses[0] rec.addresses[1].address == multiAddresses[1] + + test "Public address mapping": + let + seckey = PrivateKey.random(ECDSA, rng[]).get() + multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()] + multiAddresses2 = @[MultiAddress.init("/ip4/8.8.8.8/tcp/33").tryGet()] + + proc addressMapper(input: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} = + check input == multiAddresses + await sleepAsync(0.seconds) + return multiAddresses2 + var peerInfo = PeerInfo.new(seckey, multiAddresses, addressMappers = @[addressMapper]) + waitFor peerInfo.update() + check peerInfo.addrs == multiAddresses2 diff --git a/tests/testrelayv2.nim b/tests/testrelayv2.nim index 8565b36..16d37d9 100644 --- a/tests/testrelayv2.nim +++ b/tests/testrelayv2.nim @@ -118,7 +118,6 @@ suite "Circuit Relay V2": asyncTeardown: checkTrackers() var - addrs {.threadvar.}: MultiAddress customProtoCodec {.threadvar.}: string proto {.threadvar.}: LPProtocol ttl {.threadvar.}: int @@ -145,9 +144,6 @@ suite "Circuit Relay V2": src = createSwitch(srcCl) dst = createSwitch(dstCl) rel = newStandardSwitch() - addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & - $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $dst.peerInfo.peerId).get() asyncTest "Connection succeed": proto.handler = proc(conn: Connection, proto: string) {.async.} = @@ -167,6 +163,10 @@ suite "Circuit Relay V2": await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -200,6 +200,10 @@ suite "Circuit Relay V2": await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -245,6 +249,10 @@ take to the ship.""") await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -277,6 +285,10 @@ take to the ship.""") await src.start() await dst.start() + let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) @@ -308,11 +320,6 @@ take to the ship.""") rel2Cl = RelayClient.new(canHop = true) rel2 = createSwitch(rel2Cl) rv2 = Relay.new() - addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & - $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & - $rel2.peerInfo.peerId & "/p2p/" & - $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" & - $dst.peerInfo.peerId).get() ] rv2.setup(rel) rel.mount(rv2) dst.mount(proto) @@ -321,6 +328,13 @@ take to the ship.""") await src.start() await dst.start() + let + addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" & + $rel.peerInfo.peerId & "/p2p-circuit/p2p/" & + $rel2.peerInfo.peerId & "/p2p/" & + $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" & + $dst.peerInfo.peerId).get() ] + await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await rel2.connect(rel.peerInfo.peerId, rel.peerInfo.addrs) await dst.connect(rel2.peerInfo.peerId, rel2.peerInfo.addrs) @@ -367,6 +381,16 @@ take to the ship.""") switchA = createSwitch(clientA) switchB = createSwitch(clientB) switchC = createSwitch(clientC) + + switchA.mount(protoBCA) + switchB.mount(protoCAB) + switchC.mount(protoABC) + + await switchA.start() + await switchB.start() + await switchC.start() + + let addrsABC = MultiAddress.init($switchB.peerInfo.addrs[0] & "/p2p/" & $switchB.peerInfo.peerId & "/p2p-circuit/p2p/" & $switchC.peerInfo.peerId).get() @@ -376,13 +400,6 @@ take to the ship.""") addrsCAB = MultiAddress.init($switchA.peerInfo.addrs[0] & "/p2p/" & $switchA.peerInfo.peerId & "/p2p-circuit/p2p/" & $switchB.peerInfo.peerId).get() - switchA.mount(protoBCA) - switchB.mount(protoCAB) - switchC.mount(protoABC) - - await switchA.start() - await switchB.start() - await switchC.start() await switchA.connect(switchB.peerInfo.peerId, switchB.peerInfo.addrs) await switchB.connect(switchC.peerInfo.peerId, switchC.peerInfo.addrs) From a086fcba7201f486cab90ab69d637698d8b82482 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 20 Oct 2022 14:52:02 +0200 Subject: [PATCH 18/20] Remove shallow copies (#782) --- libp2p/crypto/minasn1.nim | 32 ++++++++++++-------------------- libp2p/multiaddress.nim | 16 ++++------------ libp2p/peerid.nim | 2 +- libp2p/protobuf/minprotobuf.nim | 2 +- libp2p/vbuffer.nim | 14 +------------- 5 files changed, 19 insertions(+), 47 deletions(-) diff --git a/libp2p/crypto/minasn1.nim b/libp2p/crypto/minasn1.nim index cb30b66..4c7966d 100644 --- a/libp2p/crypto/minasn1.nim +++ b/libp2p/crypto/minasn1.nim @@ -528,8 +528,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass, index: ttag, offset: int(ab.offset), - length: 1) - shallowCopy(field.buffer, ab.buffer) + length: 1, buffer: ab.buffer) field.vbool = (b == 0xFF'u8) ab.offset += 1 return ok(field) @@ -554,8 +553,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Negative or Positive integer field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8: # Negative integer if length <= 8: @@ -579,16 +577,15 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Zero value integer field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length), vint: 0'u64) - shallowCopy(field.buffer, ab.buffer) + length: int(length), vint: 0'u64, + buffer: ab.buffer) ab.offset += int(length) return ok(field) else: # Positive integer with leading zero field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass, index: ttag, offset: int(ab.offset) + 1, - length: int(length) - 1) - shallowCopy(field.buffer, ab.buffer) + length: int(length) - 1, buffer: ab.buffer) if length <= 9: for i in 1 ..< int(length): field.vint = (field.vint shl 8) or @@ -610,8 +607,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = # Zero-length BIT STRING. field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass, index: ttag, offset: int(ab.offset + 1), - length: 0, ubits: 0) - shallowCopy(field.buffer, ab.buffer) + length: 0, ubits: 0, buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -631,8 +627,8 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass, index: ttag, offset: int(ab.offset + 1), - length: int(length - 1), ubits: int(unused)) - shallowCopy(field.buffer, ab.buffer) + length: int(length - 1), ubits: int(unused), + buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -643,8 +639,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -654,8 +649,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = return err(Asn1Error.Incorrect) field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag, - offset: int(ab.offset), length: 0) - shallowCopy(field.buffer, ab.buffer) + offset: int(ab.offset), length: 0, buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -666,8 +660,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) @@ -678,8 +671,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] = field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass, index: ttag, offset: int(ab.offset), - length: int(length)) - shallowCopy(field.buffer, ab.buffer) + length: int(length), buffer: ab.buffer) ab.offset += int(length) return ok(field) diff --git a/libp2p/multiaddress.nim b/libp2p/multiaddress.nim index fd663f9..d7ef6e9 100644 --- a/libp2p/multiaddress.nim +++ b/libp2p/multiaddress.nim @@ -516,15 +516,10 @@ proc trimRight(s: string, ch: char): string = break result = s[0..(s.high - m)] -proc shcopy*(m1: var MultiAddress, m2: MultiAddress) = - shallowCopy(m1.data.buffer, m2.data.buffer) - m1.data.offset = m2.data.offset - proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] = ## Returns MultiAddress ``ma`` protocol code. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -537,8 +532,7 @@ proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] = proc protoName*(ma: MultiAddress): MaResult[string] = ## Returns MultiAddress ``ma`` protocol name. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -555,9 +549,8 @@ proc protoArgument*(ma: MultiAddress, ## If current MultiAddress do not have argument value, then result will be ## ``0``. var header: uint64 - var vb: MultiAddress + var vb = ma var buffer: seq[byte] - shcopy(vb, ma) if vb.data.readVarint(header) == -1: err("multiaddress: Malformed binary address!") else: @@ -792,8 +785,7 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string, proc validate*(ma: MultiAddress): bool = ## Returns ``true`` if MultiAddress ``ma`` is valid. var header: uint64 - var vb: MultiAddress - shcopy(vb, ma) + var vb = ma while true: if vb.data.isEmpty(): break diff --git a/libp2p/peerid.nim b/libp2p/peerid.nim index 401fee9..e9fac8e 100644 --- a/libp2p/peerid.nim +++ b/libp2p/peerid.nim @@ -148,7 +148,7 @@ func init*(pid: var PeerId, data: string): bool = if Base58.decode(data, p, length) == Base58Status.Success: p.setLen(length) var opid: PeerId - shallowCopy(opid.data, p) + opid.data = p if opid.validate(): pid = opid result = true diff --git a/libp2p/protobuf/minprotobuf.nim b/libp2p/protobuf/minprotobuf.nim index e1e36bd..57223b0 100644 --- a/libp2p/protobuf/minprotobuf.nim +++ b/libp2p/protobuf/minprotobuf.nim @@ -124,7 +124,7 @@ proc vsizeof*(field: ProtoField): int {.inline.} = proc initProtoBuffer*(data: seq[byte], offset = 0, options: set[ProtoFlags] = {}): ProtoBuffer = ## Initialize ProtoBuffer with shallow copy of ``data``. - shallowCopy(result.buffer, data) + result.buffer = data result.offset = offset result.options = options diff --git a/libp2p/vbuffer.nim b/libp2p/vbuffer.nim index 7a834ad..0b5b9ce 100644 --- a/libp2p/vbuffer.nim +++ b/libp2p/vbuffer.nim @@ -39,21 +39,9 @@ proc len*(vb: VBuffer): int = result = len(vb.buffer) - vb.offset doAssert(result >= 0) -proc isLiteral[T](s: seq[T]): bool {.inline.} = - when defined(gcOrc) or defined(gcArc): - false - else: - type - SeqHeader = object - length, reserved: int - (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 - proc initVBuffer*(data: seq[byte], offset = 0): VBuffer = ## Initialize VBuffer with shallow copy of ``data``. - if isLiteral(data): - result.buffer = data - else: - shallowCopy(result.buffer, data) + result.buffer = data result.offset = offset proc initVBuffer*(data: openArray[byte], offset = 0): VBuffer = From 2e12c7ab730eaef6f5e870d1542d64ff7760b005 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 21 Oct 2022 16:59:53 +0200 Subject: [PATCH 19/20] Temporarily remove failing test (#788) --- tests/pubsub/testpubsub.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pubsub/testpubsub.nim b/tests/pubsub/testpubsub.nim index 08b9aba..0e910cb 100644 --- a/tests/pubsub/testpubsub.nim +++ b/tests/pubsub/testpubsub.nim @@ -2,9 +2,11 @@ import ../stublogger -import testfloodsub, - testgossipsub, - testgossipsub2, +import testfloodsub +when not defined(linux): + import testgossipsub, + testgossipsub2 +import testmcache, testtimedcache, testmessage From 4b105c6abde61cf6dbb96f2cb474c13a3a77ed2e Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 21 Oct 2022 17:00:36 +0200 Subject: [PATCH 20/20] GossipSub tutorial (#784) --- examples/tutorial_4_gossipsub.nim | 163 ++++++++++++++++++++ libp2p.nimble | 4 +- libp2p/protocols/pubsub/gossipsub/types.nim | 6 +- mkdocs.yml | 1 + 4 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 examples/tutorial_4_gossipsub.nim diff --git a/examples/tutorial_4_gossipsub.nim b/examples/tutorial_4_gossipsub.nim new file mode 100644 index 0000000..0e8274c --- /dev/null +++ b/examples/tutorial_4_gossipsub.nim @@ -0,0 +1,163 @@ +## # GossipSub +## +## In this tutorial, we'll build a simple GossipSub network +## to broadcast the metrics we built in the previous tutorial. +## +## GossipSub is used to broadcast some messages in a network, +## and allows to balance between latency, bandwidth usage, +## privacy and attack resistance. +## +## You'll find a good explanation on how GossipSub works +## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot +## of parameters you can tweak to adjust how GossipSub behaves but here we'll +## use the sane defaults shipped with libp2p. +## +## We'll start by creating our metric structure like previously + +import chronos +import stew/results + +import libp2p +import libp2p/protocols/pubsub/rpc/messages + +type + Metric = object + name: string + value: float + + MetricList = object + hostname: string + metrics: seq[Metric] + +{.push raises: [].} + +proc encode(m: Metric): ProtoBuffer = + result = initProtoBuffer() + result.write(1, m.name) + result.write(2, m.value) + result.finish() + +proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] = + var res: Metric + let pb = initProtoBuffer(buf) + discard ? pb.getField(1, res.name) + discard ? pb.getField(2, res.value) + ok(res) + +proc encode(m: MetricList): ProtoBuffer = + result = initProtoBuffer() + for metric in m.metrics: + result.write(1, metric.encode()) + result.write(2, m.hostname) + result.finish() + +proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] = + var + res: MetricList + metrics: seq[seq[byte]] + let pb = initProtoBuffer(buf) + discard ? pb.getRepeatedField(1, metrics) + + for metric in metrics: + res.metrics &= ? Metric.decode(metric) + ? pb.getRequiredField(2, res.hostname) + ok(res) + +## This is exactly like the previous structure, except that we added +## a `hostname` to distinguish where the metric is coming from. +## +## Now we'll create a small GossipSub network to broadcast the metrics, +## and collect them on one of the node. + +type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string] + +proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} = + # This procedure will handle one of the node of the network + node.gossip.addValidator(["metrics"], + proc(topic: string, message: Message): Future[ValidationResult] {.async.} = + let decoded = MetricList.decode(message.data) + if decoded.isErr: return ValidationResult.Reject + return ValidationResult.Accept + ) + # This "validator" will attach to the `metrics` topic and make sure + # that every message in this topic is valid. This allows us to stop + # propagation of invalid messages quickly in the network, and punish + # peers sending them. + + # `John` will be responsible to log the metrics, the rest of the nodes + # will just forward them in the network + if node.hostname == "John": + node.gossip.subscribe("metrics", + proc (topic: string, data: seq[byte]) {.async.} = + echo MetricList.decode(data).tryGet() + ) + else: + node.gossip.subscribe("metrics", nil) + + # Create random metrics 10 times and broadcast them + for _ in 0..<10: + await sleepAsync(500.milliseconds) + var metricList = MetricList(hostname: node.hostname) + let metricCount = rng[].generate(uint32) mod 4 + for i in 0 ..< metricCount + 1: + metricList.metrics.add(Metric( + name: "metric_" & $i, + value: float(rng[].generate(uint16)) / 1000.0 + )) + + discard await node.gossip.publish("metrics", encode(metricList).buffer) + await node.switch.stop() + +## For our main procedure, we'll create a few nodes, and connect them together. +## Note that they are not all interconnected, but GossipSub will take care of +## broadcasting to the full network nonetheless. +proc main {.async.} = + let rng = newRng() + var nodes: seq[Node] + + for hostname in ["John", "Walter", "David", "Thuy", "Amy"]: + let + switch = newStandardSwitch(rng=rng) + gossip = GossipSub.init(switch = switch, triggerSelf = true) + switch.mount(gossip) + await switch.start() + + nodes.add((switch, gossip, hostname)) + + for index, node in nodes: + # Connect to a few neighbors + for otherNodeIdx in index - 1 .. index + 2: + if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index: continue + let otherNode = nodes[otherNodeIdx] + await node.switch.connect( + otherNode.switch.peerInfo.peerId, + otherNode.switch.peerInfo.addrs) + + var allFuts: seq[Future[void]] + for node in nodes: + allFuts.add(oneNode(node, rng)) + + await allFutures(allFuts) + +waitFor(main()) + +## If you run this program, you should see something like: +## ``` +## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)]) +## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)]) +## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)]) +## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)]) +## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)]) +## ``` +## +## This is John receiving & logging everyone's metrics. +## +## ## Going further +## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams) +## and [topic params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams), +## you can achieve very different properties. +## +## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4) +## +## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub, +## and adds features such as history, spam protection, and light node friendliness. diff --git a/libp2p.nimble b/libp2p.nimble index eb661d8..568df00 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -90,6 +90,7 @@ task website, "Build the website": tutorialToMd("examples/tutorial_1_connect.nim") tutorialToMd("examples/tutorial_2_customproto.nim") tutorialToMd("examples/tutorial_3_protobuf.nim") + tutorialToMd("examples/tutorial_4_gossipsub.nim") tutorialToMd("examples/circuitrelay.nim") exec "mkdocs build" @@ -100,8 +101,9 @@ task examples_build, "Build the samples": buildSample("tutorial_1_connect", true) buildSample("tutorial_2_customproto", true) if (NimMajor, NimMinor) > (1, 2): - # This tutorial relies on post 1.4 exception tracking + # These tutorials relies on post 1.4 exception tracking buildSample("tutorial_3_protobuf", true) + buildSample("tutorial_4_gossipsub", true) # pin system # while nimble lockfile diff --git a/libp2p/protocols/pubsub/gossipsub/types.nim b/libp2p/protocols/pubsub/gossipsub/types.nim index 5142aef..5bc96ea 100644 --- a/libp2p/protocols/pubsub/gossipsub/types.nim +++ b/libp2p/protocols/pubsub/gossipsub/types.nim @@ -16,7 +16,7 @@ import chronos import std/[tables, sets] import ".."/[floodsub, peertable, mcache, pubsubpeer] import "../rpc"/[messages] -import "../../.."/[peerid, multiaddress] +import "../../.."/[peerid, multiaddress, utility] const GossipSubCodec* = "/meshsub/1.1.0" @@ -65,7 +65,7 @@ type meshFailurePenalty*: float64 invalidMessageDeliveries*: float64 - TopicParams* = object + TopicParams* {.public.} = object topicWeight*: float64 # p1 @@ -102,7 +102,7 @@ type appScore*: float64 # application specific score behaviourPenalty*: float64 # the eventual penalty score - GossipSubParams* = object + GossipSubParams* {.public.} = object explicit*: bool pruneBackoff*: Duration unsubscribeBackoff*: Duration diff --git a/mkdocs.yml b/mkdocs.yml index 57d8479..c1906f3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,5 +45,6 @@ nav: - 'Simple connection': tutorial_1_connect.md - 'Create a custom protocol': tutorial_2_customproto.md - 'Protobuf': tutorial_3_protobuf.md + - 'GossipSub': tutorial_4_gossipsub.md - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html'