Merge branch 'master' into unstable

README.md

@@ -18,10 +18,12 @@
 
 ## Introduction
 
-An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wrapper of the [Libp2p Go daemon](https://github.com/libp2p/go-libp2p).
+An implementation of [libp2p](https://libp2p.io/) in Nim.
 
 ## Project Status
-The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :)
+libp2p is now used in production by a few projects at [status](https://github.com/status-im), including [nimbus](https://github.com/status-im/nimbus-eth2).
+
+While far from complete, currently available componets are fairly stable.
 
 Check our [examples folder](/examples) to get started!
 
@@ -88,12 +90,6 @@ git clone https://github.com/status-im/nim-libp2p
 cd nim-libp2p
 nimble install
 ```
-
-### Tests
-
-#### Prerequisite
-- [Go 1.12+](https://golang.org/dl/)
-
 #### Run unit tests
 ```sh
 # run all the unit tests

docs/GO_DAEMON.md

@@ -1,56 +0,0 @@
-# Table of Contents
-- [Introduction](#introduction)
-- [Installation](#installation)
-- [Usage](#usage)
-  - [Example](#example)
-  - [Getting Started](#getting-started)
-
-# Introduction
-This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim. <br>
-For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon).
-
-# Installation
-```sh
-# clone and install dependencies
-git clone https://github.com/status-im/nim-libp2p
-cd nim-libp2p
-nimble install
-
-# perform unit tests
-nimble test
-
-# update the git submodule to install the go daemon 
-git submodule update --init --recursive
-go version
-git clone https://github.com/libp2p/go-libp2p-daemon
-cd go-libp2p-daemon
-git checkout v0.0.1
-go install ./...
-cd ..
-```
-
-# Usage 
-
-## Example
-Examples can be found in the [examples folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon)
-
-## Getting Started
-Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
-
-```bash
-nim c -r --threads:on examples\chat.nim
-```
-
-This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
-
-```bash
-./example/chat
-/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
-```
-
-You can now chat between the instances!
-
-![Chat example](https://imgur.com/caYRu8K.gif)
-
-
-

docs/api/libp2p/daemonapi.md

@@ -38,13 +38,13 @@ Examples can be found in the [examples folder](https://github.com/status-im/nim-
 Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
 
 ```bash
-nim c -r --threads:on examples\chat.nim
+nim c -r --threads:on examples/directchat.nim
 ```
 
 This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
 
 ```bash
-./example/chat
+./examples/directchat
 /connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
 ```
 

docs/tutorial/second.nim

@@ -1,23 +1,26 @@
 when not(compileOption("threads")):
   {.fatal: "Please, compile this program with the --threads:on option!".}
 
-import tables, strformat, strutils
+import tables, strformat, strutils, bearssl
-import chronos
+import chronos                              # an efficient library for async
-import ../libp2p/[switch,
+import ../libp2p/[switch,                   # manage transports, a single entry point for dialing and listening
-                  multistream,
+                  builders,                 # helper to build the switch object
-                  crypto/crypto,
+                  multistream,              # tag stream with short header to identify it
-                  protocols/identify,
+                  multicodec,               # multicodec utilities
-                  connection,
+                  crypto/crypto,            # cryptographic functions
-                  transports/transport,
+                  errors,                   # error handling utilities
-                  transports/tcptransport,
+                  protocols/identify,       # identify the peer info of a peer
-                  multiaddress,
+                  stream/connection,        # create and close stream read / write connections
-                  peerinfo,
+                  transports/transport,     # listen and dial to other peers using p2p protocol
-                  peerid,
+                  transports/tcptransport,  # listen and dial to other peers using client-server protocol
-                  protocols/protocol,
+                  multiaddress,             # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
-                  protocols/secure/secure,
+                  peerinfo,                 # manage the information of a peer, such as peer ID and public / private key
-                  protocols/secure/secio,
+                  peerid,                   # Implement how peers interact
-                  muxers/muxer,
+                  protocols/protocol,       # define the protocol base type
-                  muxers/mplex/mplex]
+                  protocols/secure/secure,  # define the protocol of secure connection
+                  protocols/secure/secio,   # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
+                  muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
+                  muxers/mplex/mplex]       # define some contants and message types for stream multiplexing
 
 const ChatCodec = "/nim-libp2p/chat/1.0.0"
 const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
@@ -37,31 +40,33 @@ type ChatProto = ref object of LPProtocol
   connected: bool         # if the node is connected to another peer
   started: bool           # if the node has started
 
-# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str)
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(MultiAddressError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
 proc readAndPrint(p: ChatProto) {.async.} =
   while true:
-    while p.connected:
+    var strData = await p.conn.readLp(1024)
-      echo cast[string](await p.conn.readLp(1024))
+    strData &= '\0'.uint8
+    var str = cast[cstring](addr strdata[0])
+    echo $p.switch.peerInfo.peerId & ": " & $str
     await sleepAsync(100.millis)
 
+proc dialPeer(p: ChatProto, address: string) {.async.} =
+  let
+    multiAddr = MultiAddress.init(address).tryGet()
+    # split the peerId part /p2p/...
+    peerIdBytes = multiAddr[multiCodec("p2p")]
+      .tryGet()
+      .protoAddress()
+      .tryGet()
+    remotePeer = PeerID.init(peerIdBytes).tryGet()
+    # split the wire address
+    ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
+    tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
+    wireAddr = ip4Addr & tcpAddr
+
+  echo &"dialing peer: {multiAddr}"
+  p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec)
+  p.connected = true
+  asyncSpawn p.readAndPrint()
+
 proc writeAndPrint(p: ChatProto) {.async.} =
   while true:
     if not p.connected:
@@ -110,21 +115,31 @@ proc writeAndPrint(p: ChatProto) {.async.} =
         await p.conn.writeLp(line)
       else:
         try:
-          if line.startsWith("/") and "ipfs" in line:
+          if line.startsWith("/") and "p2p" in line:
             await p.dialPeer(line)
         except:
           echo &"unable to dial remote peer {line}"
           echo getCurrentExceptionMsg()
 
 proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncSpawn p.writeAndPrint() # execute the async function but does not block
+  await p.writeAndPrint()
-  asyncSpawn p.readAndPrint()
 
-proc processInput(rfd: AsyncFD) {.async.} =
+proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
-  let transp = fromPipe(rfd)
+  var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
-  while true:
+
-    let a = await transp.readLine()
+  # create handler for incoming connection
-    echo "You just entered: " & a
+  proc handle(stream: Connection, proto: string) {.async.} =
+    if chatproto.connected and not chatproto.conn.closed:
+      echo "a chat session is already in progress - disconnecting!"
+      await stream.close()
+    else:
+      chatproto.conn = stream
+      chatproto.connected = true
+      await chatproto.readAndPrint()
+
+  # assign the new handler
+  chatproto.handler = handle
+  return chatproto
 
 proc readInput(wfd: AsyncFD) {.thread.} =
   ## This procedure performs reading from `stdin` and sends data over
@@ -135,7 +150,48 @@ proc readInput(wfd: AsyncFD) {.thread.} =
     let line = stdin.readLine()
     discard waitFor transp.write(line & "\r\n")
 
+proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
+  let transp = fromPipe(rfd)
+
+  let seckey = PrivateKey.random(RSA, rng[]).get()
+  var localAddress = DefaultAddr
+  while true:
+    echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
+    let a = await transp.readLine()
+    try:
+      if a.len > 0:
+        localAddress = a
+        break
+      # uise default
+      break
+    except:
+      echo "invalid address"
+      localAddress = DefaultAddr
+      continue
+
+  var switch = SwitchBuilder
+    .init()
+    .withRng(rng)
+    .withPrivateKey(seckey)
+    .withAddress(MultiAddress.init(localAddress).tryGet())
+    .build()
+
+  let chatProto = newChatProto(switch, transp)
+  switch.mount(chatProto)
+  let libp2pFuts = await switch.start()
+  chatProto.started = true
+
+  let id = $switch.peerInfo.peerId
+  echo "PeerID: " & id
+  echo "listening on: "
+  for a in switch.peerInfo.addrs:
+    echo &"{a}/p2p/{id}"
+
+  await chatProto.readWriteLoop()
+  await allFuturesThrowing(libp2pFuts)
+
 proc main() {.async.} =
+  let rng = newRng() # Singe random number source for the whole application
   let (rfd, wfd) = createAsyncPipe()
   if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
     raise newException(ValueError, "Could not initialize pipe!")
@@ -143,7 +199,7 @@ proc main() {.async.} =
   var thread: Thread[AsyncFD]
   thread.createThread(readInput, wfd)
 
-  await processInput(rfd)
+  await processInput(rfd, rng)
 
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
+when isMainModule: # isMainModule = true when the module is compiled as the main file
   waitFor(main())

examples/directchat.nim

@@ -4,7 +4,9 @@ when not(compileOption("threads")):
 import tables, strformat, strutils, bearssl
 import chronos                              # an efficient library for async
 import ../libp2p/[switch,                   # manage transports, a single entry point for dialing and listening
+                  builders,                 # helper to build the switch object
                   multistream,              # tag stream with short header to identify it
+                  multicodec,               # multicodec utilities
                   crypto/crypto,            # cryptographic functions
                   errors,                   # error handling utilities
                   protocols/identify,       # identify the peer info of a peer
@@ -38,33 +40,33 @@ type ChatProto = ref object of LPProtocol
   connected: bool         # if the node is connected to another peer
   started: bool           # if the node has started
 
-
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str).tryGet()
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(ValueError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
 proc readAndPrint(p: ChatProto) {.async.} =
   while true:
-    while p.connected:
+    var strData = await p.conn.readLp(1024)
-      # TODO: echo &"{p.id} -> "
+    strData &= '\0'.uint8
-
+    var str = cast[cstring](addr strdata[0])
-      echo cast[string](await p.conn.readLp(1024))
+    echo $p.switch.peerInfo.peerId & ": " & $str
     await sleepAsync(100.millis)
 
+proc dialPeer(p: ChatProto, address: string) {.async.} =
+  let
+    multiAddr = MultiAddress.init(address).tryGet()
+    # split the peerId part /p2p/...
+    peerIdBytes = multiAddr[multiCodec("p2p")]
+      .tryGet()
+      .protoAddress()
+      .tryGet()
+    remotePeer = PeerID.init(peerIdBytes).tryGet()
+    # split the wire address
+    ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
+    tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
+    wireAddr = ip4Addr & tcpAddr
+
+  echo &"dialing peer: {multiAddr}"
+  p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec)
+  p.connected = true
+  asyncSpawn p.readAndPrint()
+
 proc writeAndPrint(p: ChatProto) {.async.} =
   while true:
     if not p.connected:
@@ -113,15 +115,14 @@ proc writeAndPrint(p: ChatProto) {.async.} =
         await p.conn.writeLp(line)
       else:
         try:
-          if line.startsWith("/") and "ipfs" in line:
+          if line.startsWith("/") and "p2p" in line:
             await p.dialPeer(line)
         except:
           echo &"unable to dial remote peer {line}"
           echo getCurrentExceptionMsg()
 
 proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncSpawn p.writeAndPrint() # execute the async function but does not block
+  await p.writeAndPrint()
-  asyncSpawn p.readAndPrint()
 
 proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
   var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
@@ -134,6 +135,7 @@ proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
     else:
       chatproto.conn = stream
       chatproto.connected = true
+      await chatproto.readAndPrint()
 
   # assign the new handler
   chatproto.handler = handle
@@ -152,48 +154,38 @@ proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
   let transp = fromPipe(rfd)
 
   let seckey = PrivateKey.random(RSA, rng[]).get()
-  let peerInfo = PeerInfo.init(seckey)
   var localAddress = DefaultAddr
   while true:
     echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
     let a = await transp.readLine()
     try:
       if a.len > 0:
-        peerInfo.addrs.add(Multiaddress.init(a).tryGet())
+        localAddress = a
         break
-
+      # uise default
-      peerInfo.addrs.add(Multiaddress.init(localAddress).tryGet())
       break
     except:
       echo "invalid address"
       localAddress = DefaultAddr
       continue
 
-  # a constructor for building different multiplexers under various connections
+  var switch = SwitchBuilder
-  proc createMplex(conn: Connection): Muxer =
+    .init()
-    result = Mplex.init(conn)
+    .withRng(rng)
-
+    .withPrivateKey(seckey)
-  let mplexProvider = newMuxerProvider(createMplex, MplexCodec)
+    .withAddress(MultiAddress.init(localAddress).tryGet())
-  let transports = @[Transport(TcpTransport.init())]
+    .build()
-  let muxers = [(MplexCodec, mplexProvider)].toTable()
-  let identify = newIdentify(peerInfo)
-  let secureManagers = [Secure(newSecio(rng, seckey))]
-  let switch = newSwitch(peerInfo,
-                         transports,
-                         identify,
-                         muxers,
-                         secureManagers)
 
   let chatProto = newChatProto(switch, transp)
   switch.mount(chatProto)
   let libp2pFuts = await switch.start()
   chatProto.started = true
 
-  let id = $peerInfo.peerId
+  let id = $switch.peerInfo.peerId
   echo "PeerID: " & id
   echo "listening on: "
-  for a in peerInfo.addrs:
+  for a in switch.peerInfo.addrs:
-    echo &"{a}/ipfs/{id}"
+    echo &"{a}/p2p/{id}"
 
   await chatProto.readWriteLoop()
   await allFuturesThrowing(libp2pFuts)