From 40e21e337564fa3fd0345586adb002411a0dd9a2 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 11 Jun 2021 16:33:47 -0600 Subject: [PATCH 1/3] Cleanup testinterop imports (#589) * cleanup exports/imports, specially for testinterop * export pubsub --- libp2p.nim | 5 +++-- libp2p/protocols/pubsub.nim | 3 +++ tests/testinterop.nim | 29 +++-------------------------- 3 files changed, 9 insertions(+), 28 deletions(-) create mode 100644 libp2p/protocols/pubsub.nim diff --git a/libp2p.nim b/libp2p.nim index 668fbb8..a07bbf0 100644 --- a/libp2p.nim +++ b/libp2p.nim @@ -27,7 +27,8 @@ import peerinfo, multiaddress, builders, - crypto/crypto] + crypto/crypto, + protocols/pubsub] import bearssl @@ -36,4 +37,4 @@ export connection, multiaddress, crypto, lpstream, bufferstream, bearssl, muxer, mplex, transport, tcptransport, noise, errors, cid, multihash, - multicodec, builders + multicodec, builders, pubsub diff --git a/libp2p/protocols/pubsub.nim b/libp2p/protocols/pubsub.nim new file mode 100644 index 0000000..c094c68 --- /dev/null +++ b/libp2p/protocols/pubsub.nim @@ -0,0 +1,3 @@ +import ./pubsub/[pubsub, floodsub, gossipsub] + +export pubsub, floodsub, gossipsub diff --git a/tests/testinterop.nim b/tests/testinterop.nim index 338c1bd..833a4c7 100644 --- a/tests/testinterop.nim +++ b/tests/testinterop.nim @@ -1,35 +1,12 @@ import options, tables import chronos, chronicles, stew/byteutils import helpers -import ../libp2p/[daemon/daemonapi, - protobuf/minprotobuf, - vbuffer, - multiaddress, - multicodec, - cid, - varint, - multihash, - builders, - peerid, - peerinfo, - switch, - stream/connection, - muxers/muxer, - crypto/crypto, - muxers/mplex/mplex, - muxers/muxer, - protocols/protocol, - protocols/identify, - transports/transport, - transports/tcptransport, - protocols/secure/secure, - protocols/pubsub/pubsub, - protocols/pubsub/floodsub, - protocols/pubsub/gossipsub] +import ../libp2p +import ../libp2p/[daemon/daemonapi, varint] type # TODO: Unify both PeerInfo structs - NativePeerInfo = peerinfo.PeerInfo + NativePeerInfo = libp2p.PeerInfo DaemonPeerInfo = daemonapi.PeerInfo proc writeLp*(s: StreamTransport, msg: string | seq[byte]): Future[int] {.gcsafe.} = From 4be0cdcdb421e0cca78631e42a100a4c00d32452 Mon Sep 17 00:00:00 2001 From: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com> Date: Mon, 28 Jun 2021 11:37:00 +0900 Subject: [PATCH 2/3] Docs cleanup (#557) * remove pointless go_daemon page (made no sense) * remove confusing go requirement from main readme * wip directchat fixes * use matchPartial for connect(wire) * fix directchat * revert wire changes * rewrite directchat partially * more readme updates * fix things to follow the update on blog posts --- README.md | 12 ++-- docs/GO_DAEMON.md | 56 --------------- docs/tutorial/second.nim | 152 ++++++++++++++++++++++++++------------- examples/directchat.nim | 86 ++++++++++------------ libp2p/switch.nim | 2 +- 5 files changed, 148 insertions(+), 160 deletions(-) delete mode 100644 docs/GO_DAEMON.md diff --git a/README.md b/README.md index d1d9b14..2a28afd 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,12 @@ ## Introduction -An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wrapper of the [Libp2p Go daemon](https://github.com/libp2p/go-libp2p). +An implementation of [libp2p](https://libp2p.io/) in Nim. ## Project Status -The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :) +libp2p is now used in production by a few projects at [status](https://github.com/status-im), including [nimbus](https://github.com/status-im/nimbus-eth2). + +While far from complete, currently available componets are fairly stable. Check our [examples folder](/examples) to get started! @@ -88,12 +90,6 @@ git clone https://github.com/status-im/nim-libp2p cd nim-libp2p nimble install ``` - -### Tests - -#### Prerequisite -- [Go 1.12+](https://golang.org/dl/) - #### Run unit tests ```sh # run all the unit tests diff --git a/docs/GO_DAEMON.md b/docs/GO_DAEMON.md deleted file mode 100644 index ebddd31..0000000 --- a/docs/GO_DAEMON.md +++ /dev/null @@ -1,56 +0,0 @@ -# Table of Contents -- [Introduction](#introduction) -- [Installation](#installation) -- [Usage](#usage) - - [Example](#example) - - [Getting Started](#getting-started) - -# Introduction -This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim.
-For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon). - -# Installation -```sh -# clone and install dependencies -git clone https://github.com/status-im/nim-libp2p -cd nim-libp2p -nimble install - -# perform unit tests -nimble test - -# update the git submodule to install the go daemon -git submodule update --init --recursive -go version -git clone https://github.com/libp2p/go-libp2p-daemon -cd go-libp2p-daemon -git checkout v0.0.1 -go install ./... -cd .. -``` - -# Usage - -## Example -Examples can be found in the [examples folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon) - -## Getting Started -Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim): - -```bash -nim c -r --threads:on examples\chat.nim -``` - -This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it. - -```bash -./example/chat -/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu -``` - -You can now chat between the instances! - -![Chat example](https://imgur.com/caYRu8K.gif) - - - diff --git a/docs/tutorial/second.nim b/docs/tutorial/second.nim index c910b10..197be4a 100644 --- a/docs/tutorial/second.nim +++ b/docs/tutorial/second.nim @@ -1,23 +1,26 @@ when not(compileOption("threads")): {.fatal: "Please, compile this program with the --threads:on option!".} -import tables, strformat, strutils -import chronos -import ../libp2p/[switch, - multistream, - crypto/crypto, - protocols/identify, - connection, - transports/transport, - transports/tcptransport, - multiaddress, - peerinfo, - peerid, - protocols/protocol, - protocols/secure/secure, - protocols/secure/secio, - muxers/muxer, - muxers/mplex/mplex] +import tables, strformat, strutils, bearssl +import chronos # an efficient library for async +import ../libp2p/[switch, # manage transports, a single entry point for dialing and listening + builders, # helper to build the switch object + multistream, # tag stream with short header to identify it + multicodec, # multicodec utilities + crypto/crypto, # cryptographic functions + errors, # error handling utilities + protocols/identify, # identify the peer info of a peer + stream/connection, # create and close stream read / write connections + transports/transport, # listen and dial to other peers using p2p protocol + transports/tcptransport, # listen and dial to other peers using client-server protocol + multiaddress, # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP + peerinfo, # manage the information of a peer, such as peer ID and public / private key + peerid, # Implement how peers interact + protocols/protocol, # define the protocol base type + protocols/secure/secure, # define the protocol of secure connection + protocols/secure/secio, # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS + muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection + muxers/mplex/mplex] # define some contants and message types for stream multiplexing const ChatCodec = "/nim-libp2p/chat/1.0.0" const DefaultAddr = "/ip4/127.0.0.1/tcp/55505" @@ -37,31 +40,33 @@ type ChatProto = ref object of LPProtocol connected: bool # if the node is connected to another peer started: bool # if the node has started -# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115 -proc initAddress(T: type MultiAddress, str: string): T = - let address = MultiAddress.init(str) - if IPFS.match(address) and matchPartial(multiaddress.TCP, address): - result = address - else: - raise newException(MultiAddressError, - "Invalid bootstrap node multi-address") - -proc dialPeer(p: ChatProto, address: string) {.async.} = - let multiAddr = MultiAddress.initAddress(address); - let parts = address.split("/") - let remotePeer = PeerInfo.init(parts[^1], - [multiAddr]) - - echo &"dialing peer: {multiAddr}" - p.conn = await p.switch.dial(remotePeer, ChatCodec) - p.connected = true - proc readAndPrint(p: ChatProto) {.async.} = while true: - while p.connected: - echo cast[string](await p.conn.readLp(1024)) + var strData = await p.conn.readLp(1024) + strData &= '\0'.uint8 + var str = cast[cstring](addr strdata[0]) + echo $p.switch.peerInfo.peerId & ": " & $str await sleepAsync(100.millis) +proc dialPeer(p: ChatProto, address: string) {.async.} = + let + multiAddr = MultiAddress.init(address).tryGet() + # split the peerId part /p2p/... + peerIdBytes = multiAddr[multiCodec("p2p")] + .tryGet() + .protoAddress() + .tryGet() + remotePeer = PeerID.init(peerIdBytes).tryGet() + # split the wire address + ip4Addr = multiAddr[multiCodec("ip4")].tryGet() + tcpAddr = multiAddr[multiCodec("tcp")].tryGet() + wireAddr = ip4Addr & tcpAddr + + echo &"dialing peer: {multiAddr}" + p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec) + p.connected = true + asyncSpawn p.readAndPrint() + proc writeAndPrint(p: ChatProto) {.async.} = while true: if not p.connected: @@ -110,21 +115,31 @@ proc writeAndPrint(p: ChatProto) {.async.} = await p.conn.writeLp(line) else: try: - if line.startsWith("/") and "ipfs" in line: + if line.startsWith("/") and "p2p" in line: await p.dialPeer(line) except: echo &"unable to dial remote peer {line}" echo getCurrentExceptionMsg() proc readWriteLoop(p: ChatProto) {.async.} = - asyncCheck p.writeAndPrint() # execute the async function but does not block - asyncCheck p.readAndPrint() + await p.writeAndPrint() -proc processInput(rfd: AsyncFD) {.async.} = - let transp = fromPipe(rfd) - while true: - let a = await transp.readLine() - echo "You just entered: " & a +proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto = + var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec]) + + # create handler for incoming connection + proc handle(stream: Connection, proto: string) {.async.} = + if chatproto.connected and not chatproto.conn.closed: + echo "a chat session is already in progress - disconnecting!" + await stream.close() + else: + chatproto.conn = stream + chatproto.connected = true + await chatproto.readAndPrint() + + # assign the new handler + chatproto.handler = handle + return chatproto proc readInput(wfd: AsyncFD) {.thread.} = ## This procedure performs reading from `stdin` and sends data over @@ -135,7 +150,48 @@ proc readInput(wfd: AsyncFD) {.thread.} = let line = stdin.readLine() discard waitFor transp.write(line & "\r\n") +proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} = + let transp = fromPipe(rfd) + + let seckey = PrivateKey.random(RSA, rng[]).get() + var localAddress = DefaultAddr + while true: + echo &"Type an address to bind to or Enter to use the default {DefaultAddr}" + let a = await transp.readLine() + try: + if a.len > 0: + localAddress = a + break + # uise default + break + except: + echo "invalid address" + localAddress = DefaultAddr + continue + + var switch = SwitchBuilder + .init() + .withRng(rng) + .withPrivateKey(seckey) + .withAddress(MultiAddress.init(localAddress).tryGet()) + .build() + + let chatProto = newChatProto(switch, transp) + switch.mount(chatProto) + let libp2pFuts = await switch.start() + chatProto.started = true + + let id = $switch.peerInfo.peerId + echo "PeerID: " & id + echo "listening on: " + for a in switch.peerInfo.addrs: + echo &"{a}/p2p/{id}" + + await chatProto.readWriteLoop() + await allFuturesThrowing(libp2pFuts) + proc main() {.async.} = + let rng = newRng() # Singe random number source for the whole application let (rfd, wfd) = createAsyncPipe() if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe: raise newException(ValueError, "Could not initialize pipe!") @@ -143,7 +199,7 @@ proc main() {.async.} = var thread: Thread[AsyncFD] thread.createThread(readInput, wfd) - await processInput(rfd) + await processInput(rfd, rng) -when isMainModule: # isMainModule = true when the module is compiled as the main file +when isMainModule: # isMainModule = true when the module is compiled as the main file waitFor(main()) diff --git a/examples/directchat.nim b/examples/directchat.nim index dc48aa5..197be4a 100644 --- a/examples/directchat.nim +++ b/examples/directchat.nim @@ -4,7 +4,9 @@ when not(compileOption("threads")): import tables, strformat, strutils, bearssl import chronos # an efficient library for async import ../libp2p/[switch, # manage transports, a single entry point for dialing and listening + builders, # helper to build the switch object multistream, # tag stream with short header to identify it + multicodec, # multicodec utilities crypto/crypto, # cryptographic functions errors, # error handling utilities protocols/identify, # identify the peer info of a peer @@ -38,33 +40,33 @@ type ChatProto = ref object of LPProtocol connected: bool # if the node is connected to another peer started: bool # if the node has started - -proc initAddress(T: type MultiAddress, str: string): T = - let address = MultiAddress.init(str).tryGet() - if IPFS.match(address) and matchPartial(multiaddress.TCP, address): - result = address - else: - raise newException(ValueError, - "Invalid bootstrap node multi-address") - -proc dialPeer(p: ChatProto, address: string) {.async.} = - let multiAddr = MultiAddress.initAddress(address); - let parts = address.split("/") - let remotePeer = PeerInfo.init(parts[^1], - [multiAddr]) - - echo &"dialing peer: {multiAddr}" - p.conn = await p.switch.dial(remotePeer, ChatCodec) - p.connected = true - proc readAndPrint(p: ChatProto) {.async.} = while true: - while p.connected: - # TODO: echo &"{p.id} -> " - - echo cast[string](await p.conn.readLp(1024)) + var strData = await p.conn.readLp(1024) + strData &= '\0'.uint8 + var str = cast[cstring](addr strdata[0]) + echo $p.switch.peerInfo.peerId & ": " & $str await sleepAsync(100.millis) +proc dialPeer(p: ChatProto, address: string) {.async.} = + let + multiAddr = MultiAddress.init(address).tryGet() + # split the peerId part /p2p/... + peerIdBytes = multiAddr[multiCodec("p2p")] + .tryGet() + .protoAddress() + .tryGet() + remotePeer = PeerID.init(peerIdBytes).tryGet() + # split the wire address + ip4Addr = multiAddr[multiCodec("ip4")].tryGet() + tcpAddr = multiAddr[multiCodec("tcp")].tryGet() + wireAddr = ip4Addr & tcpAddr + + echo &"dialing peer: {multiAddr}" + p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec) + p.connected = true + asyncSpawn p.readAndPrint() + proc writeAndPrint(p: ChatProto) {.async.} = while true: if not p.connected: @@ -113,15 +115,14 @@ proc writeAndPrint(p: ChatProto) {.async.} = await p.conn.writeLp(line) else: try: - if line.startsWith("/") and "ipfs" in line: + if line.startsWith("/") and "p2p" in line: await p.dialPeer(line) except: echo &"unable to dial remote peer {line}" echo getCurrentExceptionMsg() proc readWriteLoop(p: ChatProto) {.async.} = - asyncCheck p.writeAndPrint() # execute the async function but does not block - asyncCheck p.readAndPrint() + await p.writeAndPrint() proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto = var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec]) @@ -134,6 +135,7 @@ proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto = else: chatproto.conn = stream chatproto.connected = true + await chatproto.readAndPrint() # assign the new handler chatproto.handler = handle @@ -152,48 +154,38 @@ proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} = let transp = fromPipe(rfd) let seckey = PrivateKey.random(RSA, rng[]).get() - let peerInfo = PeerInfo.init(seckey) var localAddress = DefaultAddr while true: echo &"Type an address to bind to or Enter to use the default {DefaultAddr}" let a = await transp.readLine() try: if a.len > 0: - peerInfo.addrs.add(Multiaddress.init(a).tryGet()) + localAddress = a break - - peerInfo.addrs.add(Multiaddress.init(localAddress).tryGet()) + # uise default break except: echo "invalid address" localAddress = DefaultAddr continue - # a constructor for building different multiplexers under various connections - proc createMplex(conn: Connection): Muxer = - result = Mplex.init(conn) - - let mplexProvider = newMuxerProvider(createMplex, MplexCodec) - let transports = @[Transport(TcpTransport.init())] - let muxers = [(MplexCodec, mplexProvider)].toTable() - let identify = newIdentify(peerInfo) - let secureManagers = [Secure(newSecio(rng, seckey))] - let switch = newSwitch(peerInfo, - transports, - identify, - muxers, - secureManagers) + var switch = SwitchBuilder + .init() + .withRng(rng) + .withPrivateKey(seckey) + .withAddress(MultiAddress.init(localAddress).tryGet()) + .build() let chatProto = newChatProto(switch, transp) switch.mount(chatProto) let libp2pFuts = await switch.start() chatProto.started = true - let id = $peerInfo.peerId + let id = $switch.peerInfo.peerId echo "PeerID: " & id echo "listening on: " - for a in peerInfo.addrs: - echo &"{a}/ipfs/{id}" + for a in switch.peerInfo.addrs: + echo &"{a}/p2p/{id}" await chatProto.readWriteLoop() await allFuturesThrowing(libp2pFuts) diff --git a/libp2p/switch.nim b/libp2p/switch.nim index d8cd7f9..472b8bd 100644 --- a/libp2p/switch.nim +++ b/libp2p/switch.nim @@ -128,7 +128,7 @@ proc dialAndUpgrade(s: Switch, debug "Dialing canceled", msg = exc.msg, peerId raise exc except CatchableError as exc: - debug "Dialing failed", msg = exc.msg, peerId + debug "Dialing failed", msg = exc.msg, peerId, address = $a libp2p_failed_dials.inc() continue # Try the next address From 72ccd1d1f55764df35087b063ed38f688f517d6a Mon Sep 17 00:00:00 2001 From: Ben Tilford <248725+btilford@users.noreply.github.com> Date: Sun, 27 Jun 2021 22:36:57 -0600 Subject: [PATCH 3/3] Correcting paths and file/directory names (#596) --- docs/api/libp2p/daemonapi.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/libp2p/daemonapi.md b/docs/api/libp2p/daemonapi.md index ebddd31..ce2169d 100644 --- a/docs/api/libp2p/daemonapi.md +++ b/docs/api/libp2p/daemonapi.md @@ -38,13 +38,13 @@ Examples can be found in the [examples folder](https://github.com/status-im/nim- Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim): ```bash -nim c -r --threads:on examples\chat.nim +nim c -r --threads:on examples/directchat.nim ``` This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it. ```bash -./example/chat +./examples/directchat /connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu ```