Merge remote-tracking branch 'origin/unstable'

.github/workflows/bumper.yml

@@ -7,14 +7,21 @@ on:
   workflow_dispatch:
 
 jobs:
-  bumpNimbus:
+  bumpProjects:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [
+          { repo: status-im/nimbus-eth2, branch: unstable },
+          { repo: status-im/nwaku, branch: master },
+          { repo: status-im/nim-codex, branch: main }
+        ]
     steps:
-      - name: Clone NBC
+      - name: Clone repo
         uses: actions/checkout@v2
         with:
-          repository: status-im/nimbus-eth2
+          repository: ${{ matrix.target.repo }}
-          ref: unstable
+          ref: ${{ matrix.target.branch }}
           path: nbc
           submodules: true
           fetch-depth: 0

.github/workflows/doc.yml

@@ -63,7 +63,7 @@ jobs:
           git push origin gh-pages
 
   update_site:
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs'
     name: 'Rebuild website'
     runs-on: ubuntu-latest
     steps:
@@ -74,8 +74,12 @@ jobs:
         with:
           python-version: 3.x
 
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: 'stable'
+
       - name: Generate website
-        run: pip install mkdocs-material && mkdocs build
+        run: pip install mkdocs-material && nimble website
 
       - name: Clone the gh-pages branch
         uses: actions/checkout@v2

.gitignore

@@ -13,5 +13,6 @@ build/
 .vscode/
 .DS_Store
 tests/pubsub/testgossipsub
+examples/*.md
 nimble.develop
 nimble.paths

.pinned

@@ -1,16 +1,16 @@
-bearssl;https://github.com/status-im/nim-bearssl@#25009951ff8e0006171d566e3c7dc73a8231c2ed
+bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab
 chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
-chronos;https://github.com/status-im/nim-chronos@#41b82cdea34744148600b67a9154331b76181189
+chronos;https://github.com/status-im/nim-chronos@#9df76c39df254c7ff0cec6dec5c9f345f2819c91
-dnsclient;https://github.com/ba0f3/dnsclient.nim@#4960de2b345f567b12f09a08e9967af104ab39a3
+dnsclient;https://github.com/ba0f3/dnsclient.nim@#6647ca8bd9ffcc13adaecb9cb6453032063967db
-faststreams;https://github.com/status-im/nim-faststreams@#49e2c52eb5dda46b1c9c10d079abe7bffe6cea89
+faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29
-httputils;https://github.com/status-im/nim-http-utils@#f83fbce4d6ec7927b75be3f85e4fa905fcb69788
+httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040
 json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4
-metrics;https://github.com/status-im/nim-metrics@#9070af9c830e93e5239ddc488cd65aa6f609ba73
+metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f
 nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178
-secp256k1;https://github.com/status-im/nim-secp256k1@#5340cf188168d6afcafc8023770d880f067c0b2f
+secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f
 serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466
-stew;https://github.com/status-im/nim-stew@#598246620da5c41d0e92a8dd6aab0755381b21cd
+stew;https://github.com/status-im/nim-stew@#f2e58ba4c8da65548c824e4fa8732db9739f6505
 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
 unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db
-websock;https://github.com/status-im/nim-websock@#8a72c0f7690802753b1d59887745b1ce1f0c8b3d
+websock;https://github.com/status-im/nim-websock@#2424f2b215c0546f97d8b147e21544521c7545b0
 zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8

examples/README.md

@@ -2,5 +2,5 @@
 
 Welcome to the nim-libp2p documentation!
 
-Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and
+Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as 
 the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

examples/circuitrelay.nim

@@ -1,6 +1,14 @@
+## # Circuit Relay example
+##
+## Circuit Relay can be used when a node cannot reach another node
+## directly, but can reach it through a another node (the Relay).
+##
+## That may happen because of NAT, Firewalls, or incompatible transports.
+##
+## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/).
 import chronos, stew/byteutils
-import ../libp2p,
+import libp2p,
-       ../libp2p/protocols/relay/[relay, client]
+       libp2p/protocols/connectivity/relay/[relay, client]
 
 # Helper to create a circuit relay node
 proc createCircuitRelaySwitch(r: Relay): Switch =
@@ -40,19 +48,19 @@ proc main() {.async.} =
     swSrc = createCircuitRelaySwitch(clSrc)
     swDst = createCircuitRelaySwitch(clDst)
 
-    # Create a relay address to swDst using swRel as the relay
-    addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
-                              $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
-                              $swDst.peerInfo.peerId).get()
-
   swDst.mount(proto)
 
   await swRel.start()
   await swSrc.start()
   await swDst.start()
 
-  # Connect both Src and Dst to the relay, but not to each other.
+  let
-  await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+    # Create a relay address to swDst using swRel as the relay
+    addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
+                              $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                              $swDst.peerInfo.peerId).get()
+
+  # Connect Dst to the relay
   await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
 
   # Dst reserve a slot on the relay.

examples/directchat.nim

@@ -5,7 +5,7 @@ import
   strformat, strutils,
   stew/byteutils,
   chronos,
-  ../libp2p
+  libp2p
 
 const DefaultAddr = "/ip4/127.0.0.1/tcp/0"
 

examples/helloworld.nim

@@ -1,6 +1,6 @@
 import chronos         # an efficient library for async
 import stew/byteutils  # various utils
-import ../libp2p       # when installed through nimble, just use `import libp2p`
+import libp2p
 
 ##
 # Create our custom protocol

examples/tutorial_1_connect.md

@@ -1,108 +0,0 @@
-# Simple ping tutorial
-
-Hi all, welcome to the first nim-libp2p tutorial!
-
-!!! tips ""
-    This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
-
-To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
-
-Hope you'll find it helpful in your journey of learning. Happy coding! ;)
-
-## Before you start
-The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
-
-Install Nim via their [official website](https://nim-lang.org/install.html).
-Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
-
-You can now install the latest version of `nim-libp2p`:
-```bash
-nimble install libp2p@#master
-```
-
-## A simple ping application
-We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
-
-!!! tips ""
-    You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder!
-
-Let's create a `part1.nim`, and import our dependencies:
-```nim
-import chronos
-
-import libp2p
-import libp2p/protocols/ping
-```
-[chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
-
-Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
-```nim
-proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
-  var switch = SwitchBuilder
-    .new()
-    .withRng(rng)       # Give the application RNG
-    .withAddress(ma)    # Our local address(es)
-    .withTcpTransport() # Use TCP as transport
-    .withMplex()        # Use Mplex as muxer
-    .withNoise()        # Use Noise as secure manager
-    .build()
-
-  return switch
-```
-This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
-
-You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
-
-
-Let's now start to create our main procedure:
-```nim
-proc main() {.async, gcsafe.} =
-  let
-    rng = newRng()
-    localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
-    pingProtocol = Ping.new(rng=rng)
-```
-We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
-The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
-
-`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
-
-We can now create our two switches:
-```nim
-  let
-    switch1 = createSwitch(localAddress, rng)
-    switch2 = createSwitch(localAddress, rng)
-
-  switch1.mount(pingProtocol)
-
-  await switch1.start()
-  await switch2.start()
-```
-We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
-
-Now that we've started the nodes, they are listening for incoming peers.
-We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
-
-We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
-```nim
-  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
-```
-We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
-```nim
-  # ping the other node and echo the ping duration
-  echo "ping: ", await pingProtocol.ping(conn)
-
-  # We must close the connection ourselves when we're done with it
-  await conn.close()
-```
-
-And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
-```nim
-  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
-
-waitFor(main())
-```
-
-You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
-
-In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

examples/tutorial_1_connect.nim

@@ -0,0 +1,95 @@
+## # Simple ping tutorial
+##
+## Hi all, welcome to the first nim-libp2p tutorial!
+##
+## !!! tips ""
+##     This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
+##
+## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
+##
+## Hope you'll find it helpful in your journey of learning. Happy coding! ;)
+##
+## ## Before you start
+## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
+##
+## Install Nim via their [official website](https://nim-lang.org/install.html).
+## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
+##
+## You can now install the latest version of `nim-libp2p`:
+## ```bash
+## nimble install libp2p@#master
+## ```
+##
+## ## A simple ping application
+## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
+##
+## !!! tips ""
+##     You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder!
+##
+## Let's create a `part1.nim`, and import our dependencies:
+import chronos
+
+import libp2p
+import libp2p/protocols/ping
+
+## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
+##
+## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
+  var switch = SwitchBuilder
+    .new()
+    .withRng(rng)       # Give the application RNG
+    .withAddress(ma)    # Our local address(es)
+    .withTcpTransport() # Use TCP as transport
+    .withMplex()        # Use Mplex as muxer
+    .withNoise()        # Use Noise as secure manager
+    .build()
+
+  return switch
+
+## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
+##
+## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
+##
+##
+## Let's now start to create our main procedure:
+proc main() {.async, gcsafe.} =
+  let
+    rng = newRng()
+    localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
+    pingProtocol = Ping.new(rng=rng)
+  ## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
+  ## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
+  ##
+  ## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
+  ##
+  ## We can now create our two switches:
+  let
+    switch1 = createSwitch(localAddress, rng)
+    switch2 = createSwitch(localAddress, rng)
+
+  switch1.mount(pingProtocol)
+
+  await switch1.start()
+  await switch2.start()
+  ## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
+  ##
+  ## Now that we've started the nodes, they are listening for incoming peers.
+  ## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
+  ##
+  ## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
+  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
+  ## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
+  # ping the other node and echo the ping duration
+  echo "ping: ", await pingProtocol.ping(conn)
+
+  # We must close the connection ourselves when we're done with it
+  await conn.close()
+  ## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+waitFor(main())
+
+## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
+##
+## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

examples/tutorial_2_customproto.md

@@ -1,82 +0,0 @@
-# Custom protocol in libp2p
-
-In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
-
-We'll now look at how to create a custom protocol inside the libp2p
-
-Let's create a `part2.nim`, and import our dependencies:
-```nim
-import chronos
-import stew/byteutils
-
-import libp2p
-```
-This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
-
-Next, we'll declare our custom protocol
-```nim
-const TestCodec = "/test/proto/1.0.0"
-
-type TestProto = ref object of LPProtocol
-```
-
-We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
-
-A protocol generally has two part: and handling/server part, and a dialing/client part.
-Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
-
-Let's start with the server part:
-```nim
-proc new(T: typedesc[TestProto]): T =
-  # every incoming connections will in be handled in this closure
-  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
-    # Read up to 1024 bytes from this connection, and transform them into
-    # a string
-    echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
-    # We must close the connections ourselves when we're done with it
-    await conn.close()
-
-  return T(codecs: @[TestCodec], handler: handle)
-```
-This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
-In our handle, we simply read a message from the connection and `echo` it.
-
-We can now create our client part:
-```nim
-proc hello(p: TestProto, conn: Connection) {.async.} =
-  await conn.writeLp("Hello p2p!")
-```
-Again, pretty straight-forward, we just send a message on the connection.
-
-We can now create our main procedure:
-```nim
-proc main() {.async, gcsafe.} =
-  let
-    rng = newRng()
-    testProto = TestProto.new()
-    switch1 = newStandardSwitch(rng=rng)
-    switch2 = newStandardSwitch(rng=rng)
-
-  switch1.mount(testProto)
-
-  await switch1.start()
-  await switch2.start()
-
-  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
-
-  await testProto.hello(conn)
-
-  # We must close the connection ourselves when we're done with it
-  await conn.close()
-
-  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
-```
-
-This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
-
-We can now wrap our program by calling our main proc:
-```nim
-waitFor(main())
-```
-
-And that's it!

examples/tutorial_2_customproto.nim

@@ -0,0 +1,74 @@
+## # Custom protocol in libp2p
+##
+## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
+##
+## We'll now look at how to create a custom protocol inside the libp2p
+##
+## Let's create a `part2.nim`, and import our dependencies:
+import chronos
+import stew/byteutils
+
+import libp2p
+## This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
+##
+## Next, we'll declare our custom protocol
+const TestCodec = "/test/proto/1.0.0"
+
+type TestProto = ref object of LPProtocol
+
+## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
+##
+## A protocol generally has two part: and handling/server part, and a dialing/client part.
+## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
+##
+## Let's start with the server part:
+
+proc new(T: typedesc[TestProto]): T =
+  # every incoming connections will in be handled in this closure
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    # Read up to 1024 bytes from this connection, and transform them into
+    # a string
+    echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
+    # We must close the connections ourselves when we're done with it
+    await conn.close()
+
+  return T(codecs: @[TestCodec], handler: handle)
+
+## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
+## In our handle, we simply read a message from the connection and `echo` it.
+##
+## We can now create our client part:
+proc hello(p: TestProto, conn: Connection) {.async.} =
+  await conn.writeLp("Hello p2p!")
+
+## Again, pretty straight-forward, we just send a message on the connection.
+##
+## We can now create our main procedure:
+proc main() {.async, gcsafe.} =
+  let
+    rng = newRng()
+    testProto = TestProto.new()
+    switch1 = newStandardSwitch(rng=rng)
+    switch2 = newStandardSwitch(rng=rng)
+
+  switch1.mount(testProto)
+
+  await switch1.start()
+  await switch2.start()
+
+  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
+
+  await testProto.hello(conn)
+
+  # We must close the connection ourselves when we're done with it
+  await conn.close()
+
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
+##
+## We can now wrap our program by calling our main proc:
+waitFor(main())
+
+## And that's it!
+## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf.

examples/tutorial_3_protobuf.nim

@@ -0,0 +1,162 @@
+## # Protobuf usage
+##
+## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol.
+## Most real protocol want their messages to be structured and extensible, which is why
+## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to
+## define their message structures.
+##
+## Here, we'll create a slightly more complex protocol, which parses & generate protobuf
+## messages. Let's start by importing our dependencies, as usual:
+import chronos
+import stew/results # for Opt[T]
+
+import libp2p
+
+## ## Protobuf encoding & decoding
+## This will be the structure of our messages:
+## ```protobuf
+## message MetricList {
+##   message Metric {
+##     string name = 1;
+##     float value = 2;
+##   }
+##
+##   repeated Metric metrics = 2;
+## }
+## ```
+## We'll create our protobuf types, encoders & decoders, according to this format.
+## To create the encoders & decoders, we are going to use minprotobuf
+## (included in libp2p).
+##
+## While more modern technics
+## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization))
+## exists, minprotobuf is currently the recommended method to handle protobuf, since it has
+## been used in production extensively, and audited.
+type
+  Metric = object
+    name: string
+    value: float
+
+  MetricList = object
+    metrics: seq[Metric]
+
+{.push raises: [].}
+
+proc encode(m: Metric): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, m.name)
+  result.write(2, m.value)
+  result.finish()
+
+proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
+  var res: Metric
+  let pb = initProtoBuffer(buf)
+  # "getField" will return a Result[bool, ProtoError].
+  # The Result will hold an error if the protobuf is invalid.
+  # The Result will hold "false" if the field is missing
+  #
+  # We are just checking the error, and ignoring whether the value
+  # is present or not (default values are valid).
+  discard ? pb.getField(1, res.name)
+  discard ? pb.getField(2, res.value)
+  ok(res)
+
+proc encode(m: MetricList): ProtoBuffer =
+  result = initProtoBuffer()
+  for metric in m.metrics:
+    result.write(1, metric.encode())
+  result.finish()
+
+proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
+  var
+    res: MetricList
+    metrics: seq[seq[byte]]
+  let pb = initProtoBuffer(buf)
+  discard ? pb.getRepeatedField(1, metrics)
+
+  for metric in metrics:
+    res.metrics &= ? Metric.decode(metric)
+  ok(res)
+
+## ## Results instead of exceptions
+## As you can see, this part of the program also uses Results instead of exceptions for error handling.
+## We start by `{.push raises: [].}`, which will prevent every non-async function from raising
+## exceptions.
+##
+## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey
+## errors to function callers. A `Result[T, E]` will either hold a valid result of type
+## T, or an error of type E.
+##
+## You can check if the call succeeded by using `res.isOk`, and then get the
+## value using `res.value` or the error by using `res.error`.
+##
+## Another useful tool is `?`, which will unpack a Result if it succeeded,
+## or if it failed, exit the current procedure returning the error.
+##
+## nim-result is packed with other functionalities that you'll find in the
+## nim-result repository.
+##
+## Results and exception are generally interchangeable, but have different semantics
+## that you may or may not prefer.
+##
+## ## Creating the protocol
+## We'll next create a protocol, like in the last tutorial, to request these metrics from our host
+type
+  MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.}
+  MetricProto = ref object of LPProtocol
+    metricGetter: MetricCallback
+
+proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto =
+  let res = MetricProto(metricGetter: cb)
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    let
+      metrics = await res.metricGetter()
+      asProtobuf = metrics.encode()
+    await conn.writeLp(asProtobuf.buffer)
+    await conn.close()
+
+  res.codecs = @["/metric-getter/1.0.0"]
+  res.handler = handle
+  return res
+
+proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} =
+  let protobuf = await conn.readLp(2048)
+  # tryGet will raise an exception if the Result contains an error.
+  # It's useful to bridge between exception-world and result-world
+  return MetricList.decode(protobuf).tryGet()
+
+## We can now create our main procedure:
+proc main() {.async, gcsafe.} =
+  let rng = newRng()
+  proc randomMetricGenerator: Future[MetricList] {.async.} =
+    let metricCount = rng[].generate(uint32) mod 16
+    for i in 0 ..< metricCount + 1:
+      result.metrics.add(Metric(
+        name: "metric_" & $i,
+        value: float(rng[].generate(uint16)) / 1000.0
+      ))
+    return result
+  let
+    metricProto1 = MetricProto.new(randomMetricGenerator)
+    metricProto2 = MetricProto.new(randomMetricGenerator)
+    switch1 = newStandardSwitch(rng=rng)
+    switch2 = newStandardSwitch(rng=rng)
+
+  switch1.mount(metricProto1)
+
+  await switch1.start()
+  await switch2.start()
+
+  let
+    conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs)
+    metrics = await metricProto2.fetch(conn)
+  await conn.close()
+
+  for metric in metrics.metrics:
+    echo metric.name, " = ", metric.value
+
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+waitFor(main())
+
+## If you run this program, you should see random metrics being sent from the switch1 to the switch2.

examples/tutorial_4_gossipsub.nim

@@ -0,0 +1,163 @@
+## # GossipSub
+##
+## In this tutorial, we'll build a simple GossipSub network
+## to broadcast the metrics we built in the previous tutorial.
+##
+## GossipSub is used to broadcast some messages in a network,
+## and allows to balance between latency, bandwidth usage,
+## privacy and attack resistance.
+##
+## You'll find a good explanation on how GossipSub works
+## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot
+## of parameters you can tweak to adjust how GossipSub behaves but here we'll
+## use the sane defaults shipped with libp2p.
+##
+## We'll start by creating our metric structure like previously
+
+import chronos
+import stew/results
+
+import libp2p
+import libp2p/protocols/pubsub/rpc/messages
+
+type
+  Metric = object
+    name: string
+    value: float
+
+  MetricList = object
+    hostname: string
+    metrics: seq[Metric]
+
+{.push raises: [].}
+
+proc encode(m: Metric): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, m.name)
+  result.write(2, m.value)
+  result.finish()
+
+proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
+  var res: Metric
+  let pb = initProtoBuffer(buf)
+  discard ? pb.getField(1, res.name)
+  discard ? pb.getField(2, res.value)
+  ok(res)
+
+proc encode(m: MetricList): ProtoBuffer =
+  result = initProtoBuffer()
+  for metric in m.metrics:
+    result.write(1, metric.encode())
+  result.write(2, m.hostname)
+  result.finish()
+
+proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
+  var
+    res: MetricList
+    metrics: seq[seq[byte]]
+  let pb = initProtoBuffer(buf)
+  discard ? pb.getRepeatedField(1, metrics)
+
+  for metric in metrics:
+    res.metrics &= ? Metric.decode(metric)
+  ? pb.getRequiredField(2, res.hostname)
+  ok(res)
+
+## This is exactly like the previous structure, except that we added
+## a `hostname` to distinguish where the metric is coming from.
+##
+## Now we'll create a small GossipSub network to broadcast the metrics,
+## and collect them on one of the node.
+
+type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string]
+
+proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} =
+  # This procedure will handle one of the node of the network
+  node.gossip.addValidator(["metrics"],
+    proc(topic: string, message: Message): Future[ValidationResult] {.async.} =
+      let decoded = MetricList.decode(message.data)
+      if decoded.isErr: return ValidationResult.Reject
+      return ValidationResult.Accept
+  )
+  # This "validator" will attach to the `metrics` topic and make sure
+  # that every message in this topic is valid. This allows us to stop
+  # propagation of invalid messages quickly in the network, and punish
+  # peers sending them.
+
+  # `John` will be responsible to log the metrics, the rest of the nodes
+  # will just forward them in the network
+  if node.hostname == "John":
+    node.gossip.subscribe("metrics",
+      proc (topic: string, data: seq[byte]) {.async.} =
+        echo MetricList.decode(data).tryGet()
+    )
+  else:
+    node.gossip.subscribe("metrics", nil)
+
+  # Create random metrics 10 times and broadcast them
+  for _ in 0..<10:
+    await sleepAsync(500.milliseconds)
+    var metricList = MetricList(hostname: node.hostname)
+    let metricCount = rng[].generate(uint32) mod 4
+    for i in 0 ..< metricCount + 1:
+      metricList.metrics.add(Metric(
+        name: "metric_" & $i,
+        value: float(rng[].generate(uint16)) / 1000.0
+      ))
+
+    discard await node.gossip.publish("metrics", encode(metricList).buffer)
+  await node.switch.stop()
+
+## For our main procedure, we'll create a few nodes, and connect them together.
+## Note that they are not all interconnected, but GossipSub will take care of
+## broadcasting to the full network nonetheless.
+proc main {.async.} =
+  let rng = newRng()
+  var nodes: seq[Node]
+
+  for hostname in ["John", "Walter", "David", "Thuy", "Amy"]:
+    let
+      switch = newStandardSwitch(rng=rng)
+      gossip = GossipSub.init(switch = switch, triggerSelf = true)
+    switch.mount(gossip)
+    await switch.start()
+
+    nodes.add((switch, gossip, hostname))
+
+  for index, node in nodes:
+    # Connect to a few neighbors
+    for otherNodeIdx in index - 1 .. index + 2:
+      if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index: continue
+      let otherNode = nodes[otherNodeIdx]
+      await node.switch.connect(
+        otherNode.switch.peerInfo.peerId,
+        otherNode.switch.peerInfo.addrs)
+
+  var allFuts: seq[Future[void]]
+  for node in nodes:
+    allFuts.add(oneNode(node, rng))
+
+  await allFutures(allFuts)
+
+waitFor(main())
+
+## If you run this program, you should see something like:
+## ```
+## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)])
+## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)])
+## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)])
+## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)])
+## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)])
+## ```
+##
+## This is John receiving & logging everyone's metrics.
+##
+## ## Going further
+## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams)
+## and [topic params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams),
+## you can achieve very different properties.
+##
+## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4)
+##
+## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub,
+## and adds features such as history, spam protection, and light node friendliness.

libp2p.nimble

@@ -9,7 +9,7 @@ skipDirs      = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
 
 requires "nim >= 1.2.0",
          "nimcrypto >= 0.4.1",
-         "dnsclient >= 0.1.2",
+         "dnsclient >= 0.3.0 & < 0.4.0",
          "bearssl >= 0.1.4",
          "chronicles >= 0.10.2",
          "chronos >= 3.0.6",
@@ -32,16 +32,16 @@ proc runTest(filename: string, verify: bool = true, sign: bool = true,
   rmFile "tests/" & filename.toExe
 
 proc buildSample(filename: string, run = false) =
-  var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off "
+  var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. "
   excstr.add(" examples/" & filename)
   exec excstr
   if run:
     exec "./examples/" & filename.toExe
   rmFile "examples/" & filename.toExe
 
-proc buildTutorial(filename: string) =
+proc tutorialToMd(filename: string) =
-  discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " &
+  let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim "
-    " nim --verbosity:0 --hints:off c -"
+  writeFile(filename.replace(".nim", ".md"), markdown)
 
 task testnative, "Runs libp2p native tests":
   runTest("testnative")
@@ -86,12 +86,24 @@ task test_slim, "Runs the (slimmed down) test suite":
   exec "nimble testfilter"
   exec "nimble examples_build"
 
+task website, "Build the website":
+  tutorialToMd("examples/tutorial_1_connect.nim")
+  tutorialToMd("examples/tutorial_2_customproto.nim")
+  tutorialToMd("examples/tutorial_3_protobuf.nim")
+  tutorialToMd("examples/tutorial_4_gossipsub.nim")
+  tutorialToMd("examples/circuitrelay.nim")
+  exec "mkdocs build"
+
 task examples_build, "Build the samples":
   buildSample("directchat")
   buildSample("helloworld", true)
   buildSample("circuitrelay", true)
-  buildTutorial("examples/tutorial_1_connect.md")
+  buildSample("tutorial_1_connect", true)
-  buildTutorial("examples/tutorial_2_customproto.md")
+  buildSample("tutorial_2_customproto", true)
+  if (NimMajor, NimMinor) > (1, 2):
+    # These tutorials relies on post 1.4 exception tracking
+    buildSample("tutorial_3_protobuf", true)
+    buildSample("tutorial_4_gossipsub", true)
 
 # pin system
 # while nimble lockfile

libp2p/builders.nim

@@ -26,8 +26,8 @@ import
   switch, peerid, peerinfo, stream/connection, multiaddress,
   crypto/crypto, transports/[transport, tcptransport],
   muxers/[muxer, mplex/mplex, yamux/yamux],
-  protocols/[identify, secure/secure, secure/noise, autonat],
+  protocols/[identify, secure/secure, secure/noise, rendezvous],
-  protocols/relay/[relay, client, rtransport],
+  protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport],
   connmanager, upgrademngrs/muxedupgrade,
   nameresolving/nameresolver,
   errors, utility
@@ -60,6 +60,7 @@ type
     peerStoreCapacity: Option[int]
     autonat: bool
     circuitRelay: Relay
+    rdv: RendezVous
 
 proc new*(T: type[SwitchBuilder]): T {.public.} =
   ## Creates a SwitchBuilder
@@ -194,6 +195,10 @@ proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder
   b.circuitRelay = r
   b
 
+proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder =
+  b.rdv = rdv
+  b
+
 proc build*(b: SwitchBuilder): Switch
   {.raises: [Defect, LPError], public.} =
 
@@ -261,6 +266,10 @@ proc build*(b: SwitchBuilder): Switch
     b.circuitRelay.setup(switch)
     switch.mount(b.circuitRelay)
 
+  if not isNil(b.rdv):
+    b.rdv.setup(switch)
+    switch.mount(b.rdv)
+
   return switch
 
 proc newStandardSwitch*(

libp2p/crypto/curve25519.nim

@@ -31,7 +31,6 @@ const
 type
   Curve25519* = object
   Curve25519Key* = array[Curve25519KeySize, byte]
-  pcuchar = ptr char
   Curve25519Error* = enum
     Curver25519GenError
 

libp2p/crypto/minasn1.nim

@@ -528,8 +528,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
 
         field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass,
                           index: ttag, offset: int(ab.offset),
-                          length: 1)
+                          length: 1, buffer: ab.buffer)
-        shallowCopy(field.buffer, ab.buffer)
         field.vbool = (b == 0xFF'u8)
         ab.offset += 1
         return ok(field)
@@ -554,8 +553,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
           # Negative or Positive integer
           field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
                             index: ttag, offset: int(ab.offset),
-                            length: int(length))
+                            length: int(length), buffer: ab.buffer)
-          shallowCopy(field.buffer, ab.buffer)
           if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8:
             # Negative integer
             if length <= 8:
@@ -579,16 +577,15 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
             # Zero value integer
             field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
                               index: ttag, offset: int(ab.offset),
-                              length: int(length), vint: 0'u64)
+                              length: int(length), vint: 0'u64,
-            shallowCopy(field.buffer, ab.buffer)
+                              buffer: ab.buffer)
             ab.offset += int(length)
             return ok(field)
           else:
             # Positive integer with leading zero
             field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
                               index: ttag, offset: int(ab.offset) + 1,
-                              length: int(length) - 1)
+                              length: int(length) - 1, buffer: ab.buffer)
-            shallowCopy(field.buffer, ab.buffer)
             if length <= 9:
               for i in 1 ..< int(length):
                 field.vint = (field.vint shl 8) or
@@ -610,8 +607,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
             # Zero-length BIT STRING.
             field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
                               index: ttag, offset: int(ab.offset + 1),
-                              length: 0, ubits: 0)
+                              length: 0, ubits: 0, buffer: ab.buffer)
-            shallowCopy(field.buffer, ab.buffer)
             ab.offset += int(length)
             return ok(field)
 
@@ -631,8 +627,8 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
 
           field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
                             index: ttag, offset: int(ab.offset + 1),
-                            length: int(length - 1), ubits: int(unused))
+                            length: int(length - 1), ubits: int(unused),
-          shallowCopy(field.buffer, ab.buffer)
+                            buffer: ab.buffer)
           ab.offset += int(length)
           return ok(field)
 
@@ -643,8 +639,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
 
         field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass,
                           index: ttag, offset: int(ab.offset),
-                          length: int(length))
+                          length: int(length), buffer: ab.buffer)
-        shallowCopy(field.buffer, ab.buffer)
         ab.offset += int(length)
         return ok(field)
 
@@ -654,8 +649,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
           return err(Asn1Error.Incorrect)
 
         field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag,
-                          offset: int(ab.offset), length: 0)
+                          offset: int(ab.offset), length: 0, buffer: ab.buffer)
-        shallowCopy(field.buffer, ab.buffer)
         ab.offset += int(length)
         return ok(field)
 
@@ -666,8 +660,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
 
         field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass,
                           index: ttag, offset: int(ab.offset),
-                          length: int(length))
+                          length: int(length), buffer: ab.buffer)
-        shallowCopy(field.buffer, ab.buffer)
         ab.offset += int(length)
         return ok(field)
 
@@ -678,8 +671,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
 
         field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass,
                           index: ttag, offset: int(ab.offset),
-                          length: int(length))
+                          length: int(length), buffer: ab.buffer)
-        shallowCopy(field.buffer, ab.buffer)
         ab.offset += int(length)
         return ok(field)
 

libp2p/dial.nim

@@ -13,10 +13,13 @@ else:
   {.push raises: [].}
 
 import chronos
+import stew/results
 import peerid,
        stream/connection,
        transports/transport
 
+export results
+
 type
   Dial* = ref object of RootObj
 
@@ -31,6 +34,13 @@ method connect*(
 
   doAssert(false, "Not implemented!")
 
+method connect*(
+  self: Dial,
+  addrs: seq[MultiAddress]): Future[PeerId] {.async, base.} =
+  ## Connects to a peer and retrieve its PeerId
+
+  doAssert(false, "Not implemented!")
+
 method dial*(
   self: Dial,
   peerId: PeerId,
@@ -62,5 +72,5 @@ method addTransport*(
 method tryDial*(
   self: Dial,
   peerId: PeerId,
-  addrs: seq[MultiAddress]): Future[MultiAddress] {.async, base.} =
+  addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} =
   doAssert(false, "Not implemented!")

libp2p/dialer.nim

@@ -7,8 +7,9 @@
 # This file may not be copied, modified, or distributed except according to
 # those terms.
 
-import std/[sugar, tables]
+import std/[sugar, tables, sequtils]
 
+import stew/results
 import pkg/[chronos,
             chronicles,
             metrics]
@@ -16,6 +17,7 @@ import pkg/[chronos,
 import dial,
        peerid,
        peerinfo,
+       multicodec,
        multistream,
        connmanager,
        stream/connection,
@@ -24,7 +26,7 @@ import dial,
        upgrademngrs/upgrade,
        errors
 
-export dial, errors
+export dial, errors, results
 
 logScope:
   topics = "libp2p dialer"
@@ -47,35 +49,25 @@ type
 
 proc dialAndUpgrade(
   self: Dialer,
-  peerId: PeerId,
+  peerId: Opt[PeerId],
-  addrs: seq[MultiAddress]):
+  hostname: string,
+  address: MultiAddress):
   Future[Connection] {.async.} =
-  debug "Dialing peer", peerId
 
-  for address in addrs:      # for each address
-    let
-      hostname = address.getHostname()
-      resolvedAddresses =
-        if isNil(self.nameResolver): @[address]
-        else: await self.nameResolver.resolveMAddress(address)
-
-    for a in resolvedAddresses:      # for each resolved address
   for transport in self.transports: # for each transport
-        if transport.handles(a):   # check if it can dial it
+    if transport.handles(address):   # check if it can dial it
-          trace "Dialing address", address = $a, peerId, hostname
+      trace "Dialing address", address, peerId, hostname
-          let dialed = try:
+      let dialed =
+        try:
           libp2p_total_dial_attempts.inc()
-              await transport.dial(hostname, a)
+          await transport.dial(hostname, address)
         except CancelledError as exc:
           debug "Dialing canceled", msg = exc.msg, peerId
           raise exc
         except CatchableError as exc:
           debug "Dialing failed", msg = exc.msg, peerId
           libp2p_failed_dials.inc()
-              continue # Try the next address
+          return nil # Try the next address
-
-          # make sure to assign the peer to the connection
-          dialed.peerId = peerId
 
       # also keep track of the connection's bottom unsafe transport direction
       # required by gossipsub scoring
@@ -83,38 +75,96 @@ proc dialAndUpgrade(
 
       libp2p_successful_dials.inc()
 
-          let conn = try:
+      let conn =
-              await transport.upgradeOutgoing(dialed)
+        try:
+          await transport.upgradeOutgoing(dialed, peerId)
         except CatchableError as exc:
           # If we failed to establish the connection through one transport,
           # we won't succeeded through another - no use in trying again
-              # TODO we should try another address though
           await dialed.close()
           debug "Upgrade failed", msg = exc.msg, peerId
           if exc isnot CancelledError:
             libp2p_failed_upgrades_outgoing.inc()
-              raise exc
+
+          # Try other address
+          return nil
 
       doAssert not isNil(conn), "connection died after upgradeOutgoing"
       debug "Dial successful", conn, peerId = conn.peerId
       return conn
+  return nil
+
+proc expandDnsAddr(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} =
+
+  if not DNSADDR.matchPartial(address): return @[(address, peerId)]
+  if isNil(self.nameResolver):
+    info "Can't resolve DNSADDR without NameResolver", ma=address
+    return @[]
+
+  let
+    toResolve =
+      if peerId.isSome:
+        address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet()
+      else:
+        address
+    resolved = await self.nameResolver.resolveDnsAddr(toResolve)
+
+  for resolvedAddress in resolved:
+    let lastPart = resolvedAddress[^1].tryGet()
+    if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")):
+      let
+        peerIdBytes = lastPart.protoArgument().tryGet()
+        addrPeerId = PeerId.init(peerIdBytes).tryGet()
+      result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId)))
+    else:
+      result.add((resolvedAddress, peerId))
+
+proc dialAndUpgrade(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  addrs: seq[MultiAddress]):
+  Future[Connection] {.async.} =
+
+  debug "Dialing peer", peerId
+
+  for rawAddress in addrs:
+    # resolve potential dnsaddr
+    let addresses = await self.expandDnsAddr(peerId, rawAddress)
+
+    for (expandedAddress, addrPeerId) in addresses:
+      # DNS resolution
+      let
+        hostname = expandedAddress.getHostname()
+        resolvedAddresses =
+          if isNil(self.nameResolver): @[expandedAddress]
+          else: await self.nameResolver.resolveMAddress(expandedAddress)
+
+      for resolvedAddress in resolvedAddresses:
+        result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress)
+        if not isNil(result):
+          return result
 
 proc internalConnect(
   self: Dialer,
-  peerId: PeerId,
+  peerId: Opt[PeerId],
   addrs: seq[MultiAddress],
   forceDial: bool):
   Future[Connection] {.async.} =
-  if self.localPeerId == peerId:
+  if Opt.some(self.localPeerId) == peerId:
     raise newException(CatchableError, "can't dial self!")
 
   # Ensure there's only one in-flight attempt per peer
-  let lock = self.dialLock.mgetOrPut(peerId, newAsyncLock())
+  let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock())
   try:
     await lock.acquire()
 
     # Check if we have a connection already and try to reuse it
-    var conn = self.connManager.selectConn(peerId)
+    var conn =
+      if peerId.isSome: self.connManager.selectConn(peerId.get())
+      else: nil
     if conn != nil:
       if conn.atEof or conn.closed:
         # This connection should already have been removed from the connection
@@ -165,7 +215,15 @@ method connect*(
   if self.connManager.connCount(peerId) > 0:
     return
 
-  discard await self.internalConnect(peerId, addrs, forceDial)
+  discard await self.internalConnect(Opt.some(peerId), addrs, forceDial)
+
+method connect*(
+  self: Dialer,
+  addrs: seq[MultiAddress],
+  ): Future[PeerId] {.async.} =
+  ## Connects to a peer and retrieve its PeerId
+
+  return (await self.internalConnect(Opt.none(PeerId), addrs, false)).peerId
 
 proc negotiateStream(
   self: Dialer,
@@ -182,7 +240,7 @@ proc negotiateStream(
 method tryDial*(
   self: Dialer,
   peerId: PeerId,
-  addrs: seq[MultiAddress]): Future[MultiAddress] {.async.} =
+  addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} =
   ## Create a protocol stream in order to check
   ## if a connection is possible.
   ## Doesn't use the Connection Manager to save it.
@@ -190,7 +248,7 @@ method tryDial*(
 
   trace "Check if it can dial", peerId, addrs
   try:
-    let conn = await self.dialAndUpgrade(peerId, addrs)
+    let conn = await self.dialAndUpgrade(Opt.some(peerId), addrs)
     if conn.isNil():
       raise newException(DialFailedError, "No valid multiaddress")
     await conn.close()
@@ -238,7 +296,7 @@ method dial*(
 
   try:
     trace "Dialing (new)", peerId, protos
-    conn = await self.internalConnect(peerId, addrs, forceDial)
+    conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial)
     trace "Opening stream", conn
     stream = await self.connManager.getStream(conn)
 

libp2p/discovery/discoverymngr.nim

@@ -0,0 +1,163 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/sequtils
+import chronos, chronicles, stew/results
+import ../errors
+
+type
+  BaseAttr = ref object of RootObj
+    comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [Defect].}
+
+  Attribute[T] = ref object of BaseAttr
+    value: T
+
+  PeerAttributes* = object
+    attributes: seq[BaseAttr]
+
+  DiscoveryService* = distinct string
+
+proc `==`*(a, b: DiscoveryService): bool {.borrow.}
+
+proc ofType*[T](f: BaseAttr, _: type[T]): bool =
+  return f of Attribute[T]
+
+proc to*[T](f: BaseAttr, _: type[T]): T =
+  Attribute[T](f).value
+
+proc add*[T](pa: var PeerAttributes,
+             value: T) =
+  pa.attributes.add(Attribute[T](
+      value: value,
+      comparator: proc(f: BaseAttr, c: BaseAttr): bool =
+        f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T)
+    )
+  )
+
+iterator items*(pa: PeerAttributes): BaseAttr =
+  for f in pa.attributes:
+    yield f
+
+proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] =
+  for f in pa.attributes:
+    if f.ofType(T):
+      result.add(f.to(T))
+
+proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] =
+  for f in pa.attributes:
+    if f.ofType(T):
+      return Opt.some(f.to(T))
+  Opt.none(T)
+
+proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [Defect, KeyError].} =
+  pa{T}.valueOr: raise newException(KeyError, "Attritute not found")
+
+proc match*(pa, candidate: PeerAttributes): bool =
+  for f in pa.attributes:
+    block oneAttribute:
+      for field in candidate.attributes:
+        if field.comparator(field, f):
+          break oneAttribute
+      return false
+  return true
+
+type
+  PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [Defect], gcsafe.}
+
+  DiscoveryInterface* = ref object of RootObj
+    onPeerFound*: PeerFoundCallback
+    toAdvertise*: PeerAttributes
+    advertisementUpdated*: AsyncEvent
+    advertiseLoop*: Future[void]
+
+method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} =
+  doAssert(false, "Not implemented!")
+
+method advertise*(self: DiscoveryInterface) {.async, base.} =
+  doAssert(false, "Not implemented!")
+
+type
+  DiscoveryError* = object of LPError
+
+  DiscoveryQuery* = ref object
+    attr: PeerAttributes
+    peers: AsyncQueue[PeerAttributes]
+    futs: seq[Future[void]]
+
+  DiscoveryManager* = ref object
+    interfaces: seq[DiscoveryInterface]
+    queries: seq[DiscoveryQuery]
+
+proc add*(dm: DiscoveryManager, di: DiscoveryInterface) =
+  dm.interfaces &= di
+
+  di.onPeerFound = proc (pa: PeerAttributes) =
+    for query in dm.queries:
+      if query.attr.match(pa):
+        try:
+          query.peers.putNoWait(pa)
+        except AsyncQueueFullError as exc:
+          debug "Cannot push discovered peer to queue"
+
+proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery =
+  var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]())
+  for i in dm.interfaces:
+    query.futs.add(i.request(pa))
+  dm.queries.add(query)
+  dm.queries.keepItIf(it.futs.anyIt(not it.finished()))
+  return query
+
+proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
+  var pa: PeerAttributes
+  pa.add(value)
+  return dm.request(pa)
+
+proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) =
+  for i in dm.interfaces:
+    i.toAdvertise = pa
+    if i.advertiseLoop.isNil:
+      i.advertisementUpdated = newAsyncEvent()
+      i.advertiseLoop = i.advertise()
+    else:
+      i.advertisementUpdated.fire()
+
+proc advertise*[T](dm: DiscoveryManager, value: T) =
+  var pa: PeerAttributes
+  pa.add(value)
+  dm.advertise(pa)
+
+proc stop*(query: DiscoveryQuery) =
+  for r in query.futs:
+    if not r.finished(): r.cancel()
+
+proc stop*(dm: DiscoveryManager) =
+  for q in dm.queries:
+    q.stop()
+  for i in dm.interfaces:
+    if isNil(i.advertiseLoop): continue
+    i.advertiseLoop.cancel()
+
+proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} =
+  let getter = query.peers.popFirst()
+
+  try:
+    await getter or allFinished(query.futs)
+  except CancelledError as exc:
+    getter.cancel()
+    raise exc
+
+  if not finished(getter):
+    # discovery loops only finish when they don't handle the query
+    raise newException(DiscoveryError, "Unable to find any peer matching this request")
+  return await getter

libp2p/discovery/rendezvousinterface.nim

@@ -0,0 +1,77 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils
+import chronos
+import ./discoverymngr,
+       ../protocols/rendezvous,
+       ../peerid
+
+type
+  RendezVousInterface* = ref object of DiscoveryInterface
+    rdv*: RendezVous
+    timeToRequest: Duration
+    timeToAdvertise: Duration
+
+  RdvNamespace* = distinct string
+
+proc `==`*(a, b: RdvNamespace): bool {.borrow.}
+
+method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} =
+  var namespace = ""
+  for attr in pa:
+    if attr.ofType(RdvNamespace):
+      namespace = string attr.to(RdvNamespace)
+    elif attr.ofType(DiscoveryService):
+      namespace = string attr.to(DiscoveryService)
+    elif attr.ofType(PeerId):
+      namespace = $attr.to(PeerId)
+    else:
+      # unhandled type
+      return
+  while true:
+    for pr in await self.rdv.request(namespace):
+      var peer: PeerAttributes
+      peer.add(pr.peerId)
+      for address in pr.addresses:
+        peer.add(address.address)
+
+      peer.add(DiscoveryService(namespace))
+      peer.add(RdvNamespace(namespace))
+      self.onPeerFound(peer)
+
+    await sleepAsync(self.timeToRequest)
+
+method advertise*(self: RendezVousInterface) {.async.} =
+  while true:
+    var toAdvertise: seq[string]
+    for attr in self.toAdvertise:
+      if attr.ofType(RdvNamespace):
+        toAdvertise.add string attr.to(RdvNamespace)
+      elif attr.ofType(DiscoveryService):
+        toAdvertise.add string attr.to(DiscoveryService)
+      elif attr.ofType(PeerId):
+        toAdvertise.add $attr.to(PeerId)
+
+    self.advertisementUpdated.clear()
+    for toAdv in toAdvertise:
+      await self.rdv.advertise(toAdv, self.timeToAdvertise)
+
+    await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
+
+proc new*(T: typedesc[RendezVousInterface],
+          rdv: RendezVous,
+          ttr: Duration = 1.minutes,
+          tta: Duration = MinimumDuration): RendezVousInterface =
+  T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta)

libp2p/multiaddress.nim

@@ -222,6 +222,40 @@ proc onionVB(vb: var VBuffer): bool =
   if vb.readArray(buf) == 12:
     result = true
 
+proc onion3StB(s: string, vb: var VBuffer): bool =
+  try:
+    var parts = s.split(':')
+    if len(parts) != 2:
+      return false
+    if len(parts[0]) != 56:
+      return false
+    var address = Base32Lower.decode(parts[0].toLowerAscii())
+    var nport = parseInt(parts[1])
+    if (nport > 0 and nport < 65536) and len(address) == 35:
+      address.setLen(37)
+      address[35] = cast[byte]((nport shr 8) and 0xFF)
+      address[36] = cast[byte](nport and 0xFF)
+      vb.writeArray(address)
+      result = true
+  except:
+    discard
+
+proc onion3BtS(vb: var VBuffer, s: var string): bool =
+  ## ONION address bufferToString() implementation.
+  var buf: array[37, byte]
+  if vb.readArray(buf) == 37:
+    var nport = (cast[uint16](buf[35]) shl 8) or cast[uint16](buf[36])
+    s = Base32Lower.encode(buf.toOpenArray(0, 34))
+    s.add(":")
+    s.add($nport)
+    result = true
+
+proc onion3VB(vb: var VBuffer): bool =
+  ## ONION address validateBuffer() implementation.
+  var buf: array[37, byte]
+  if vb.readArray(buf) == 37:
+    result = true
+
 proc unixStB(s: string, vb: var VBuffer): bool =
   ## Unix socket name stringToBuffer() implementation.
   if len(s) > 0:
@@ -310,6 +344,11 @@ const
     bufferToString: onionBtS,
     validateBuffer: onionVB
   )
+  TranscoderOnion3* = Transcoder(
+    stringToBuffer: onion3StB,
+    bufferToString: onion3BtS,
+    validateBuffer: onion3VB
+  )
   TranscoderDNS* = Transcoder(
     stringToBuffer: dnsStB,
     bufferToString: dnsBtS,
@@ -363,6 +402,10 @@ const
       mcodec: multiCodec("onion"), kind: Fixed, size: 10,
       coder: TranscoderOnion
     ),
+    MAProtocol(
+      mcodec: multiCodec("onion3"), kind: Fixed, size: 37,
+      coder: TranscoderOnion3
+    ),
     MAProtocol(
       mcodec: multiCodec("ws"), kind: Marker, size: 0
     ),
@@ -473,15 +516,10 @@ proc trimRight(s: string, ch: char): string =
       break
   result = s[0..(s.high - m)]
 
-proc shcopy*(m1: var MultiAddress, m2: MultiAddress) =
-  shallowCopy(m1.data.buffer, m2.data.buffer)
-  m1.data.offset = m2.data.offset
-
 proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
   ## Returns MultiAddress ``ma`` protocol code.
   var header: uint64
-  var vb: MultiAddress
+  var vb = ma
-  shcopy(vb, ma)
   if vb.data.readVarint(header) == -1:
     err("multiaddress: Malformed binary address!")
   else:
@@ -494,8 +532,7 @@ proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
 proc protoName*(ma: MultiAddress): MaResult[string] =
   ## Returns MultiAddress ``ma`` protocol name.
   var header: uint64
-  var vb: MultiAddress
+  var vb = ma
-  shcopy(vb, ma)
   if vb.data.readVarint(header) == -1:
     err("multiaddress: Malformed binary address!")
   else:
@@ -512,9 +549,8 @@ proc protoArgument*(ma: MultiAddress,
   ## If current MultiAddress do not have argument value, then result will be
   ## ``0``.
   var header: uint64
-  var vb: MultiAddress
+  var vb = ma
   var buffer: seq[byte]
-  shcopy(vb, ma)
   if vb.data.readVarint(header) == -1:
     err("multiaddress: Malformed binary address!")
   else:
@@ -530,7 +566,7 @@ proc protoArgument*(ma: MultiAddress,
           err("multiaddress: Decoding protocol error")
         else:
           ok(res)
-      elif proto.kind in {Length, Path}:
+      elif proto.kind in {MAKind.Length, Path}:
         if vb.data.readSeq(buffer) == -1:
           err("multiaddress: Decoding protocol error")
         else:
@@ -551,6 +587,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] =
   buffer.setLen(res)
   ok(buffer)
 
+proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] =
+  ## Returns MultiAddress ``ma`` protocol address binary blob.
+  ##
+  ## If current MultiAddress do not have argument value, then result array will
+  ## be empty.
+  ma.protoAddress()
+
 proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
   var header: uint64
   var data = newSeq[byte]()
@@ -558,6 +601,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
   var vb = ma
   var res: MultiAddress
   res.data = initVBuffer()
+
+  if index < 0: return err("multiaddress: negative index gived to getPart")
+
   while offset <= index:
     if vb.data.readVarint(header) == -1:
       return err("multiaddress: Malformed binary address!")
@@ -575,7 +621,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
         res.data.writeVarint(header)
         res.data.writeArray(data)
         res.data.finish()
-    elif proto.kind in {Length, Path}:
+    elif proto.kind in {MAKind.Length, Path}:
       if vb.data.readSeq(data) == -1:
         return err("multiaddress: Decoding protocol error")
 
@@ -604,8 +650,12 @@ proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddres
     ? res.append(? ma[i])
   ok(res)
 
-proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} =
+proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} =
   ## Returns part with index ``i`` of MultiAddress ``ma``.
+  when i is BackwardsIndex:
+    let maLength = ? len(ma)
+    ma.getPart(maLength - int(i))
+  else:
     ma.getPart(i)
 
 proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} =
@@ -637,7 +687,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
 
       res.data.writeVarint(header)
       res.data.writeArray(data)
-    elif proto.kind in {Length, Path}:
+    elif proto.kind in {MAKind.Length, Path}:
       if vb.data.readSeq(data) == -1:
         yield err(MaResult[MultiAddress], "Decoding protocol error")
 
@@ -735,8 +785,7 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
 proc validate*(ma: MultiAddress): bool =
   ## Returns ``true`` if MultiAddress ``ma`` is valid.
   var header: uint64
-  var vb: MultiAddress
+  var vb = ma
-  shcopy(vb, ma)
   while true:
     if vb.data.isEmpty():
       break

libp2p/multicodec.nim

@@ -203,6 +203,7 @@ const MultiCodecList = [
   ("p2p-webrtc-star", 0x0113), # not in multicodec list
   ("p2p-webrtc-direct", 0x0114), # not in multicodec list
   ("onion", 0x01BC),
+  ("onion3", 0x01BD),
   ("p2p-circuit", 0x0122),
   ("libp2p-peer-record", 0x0301),
   ("dns", 0x35),

libp2p/muxers/mplex/lpchannel.nim

@@ -58,6 +58,8 @@ type
     initiator*: bool              # initiated remotely or locally flag
     isOpen*: bool                 # has channel been opened
     closedLocal*: bool            # has channel been closed locally
+    remoteReset*: bool            # has channel been remotely reset
+    localReset*: bool             # has channel been reset locally
     msgCode*: MessageType         # cached in/out message code
     closeCode*: MessageType       # cached in/out close code
     resetCode*: MessageType       # cached in/out reset code
@@ -103,6 +105,7 @@ proc reset*(s: LPChannel) {.async, gcsafe.} =
 
   s.isClosed = true
   s.closedLocal = true
+  s.localReset = not s.remoteReset
 
   trace "Resetting channel", s, len = s.len
 
@@ -168,6 +171,14 @@ method readOnce*(s: LPChannel,
   ## channels are blocked - in particular, this means that reading from one
   ## channel must not be done from within a callback / read handler of another
   ## or the reads will lock each other.
+  if s.remoteReset:
+    raise newLPStreamResetError()
+  if s.localReset:
+    raise newLPStreamClosedError()
+  if s.atEof():
+    raise newLPStreamRemoteClosedError()
+  if s.conn.closed:
+    raise newLPStreamConnDownError()
   try:
     let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
     when defined(libp2p_network_protocols_metrics):
@@ -184,13 +195,17 @@ method readOnce*(s: LPChannel,
     # data has been lost in s.readBuf and there's no way to gracefully recover /
     # use the channel any more
     await s.reset()
-    raise exc
+    raise newLPStreamConnDownError(exc)
 
 proc prepareWrite(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
   # prepareWrite is the slow path of writing a message - see conditions in
   # write
-  if s.closedLocal or s.conn.closed:
+  if s.remoteReset:
+    raise newLPStreamResetError()
+  if s.closedLocal:
     raise newLPStreamClosedError()
+  if s.conn.closed:
+    raise newLPStreamConnDownError()
 
   if msg.len == 0:
     return
@@ -235,7 +250,7 @@ proc completeWrite(
     trace "exception in lpchannel write handler", s, msg = exc.msg
     await s.reset()
     await s.conn.close()
-    raise exc
+    raise newLPStreamConnDownError(exc)
   finally:
     s.writes -= 1
 

libp2p/muxers/mplex/mplex.nim

@@ -183,6 +183,7 @@ method handle*(m: Mplex) {.async, gcsafe.} =
         of MessageType.CloseIn, MessageType.CloseOut:
           await channel.pushEof()
         of MessageType.ResetIn, MessageType.ResetOut:
+          channel.remoteReset = true
           await channel.reset()
   except CancelledError:
     debug "Unexpected cancellation in mplex handler", m

libp2p/muxers/yamux/yamux.nim

@@ -153,6 +153,7 @@ type
     sendQueue: seq[ToSend]
     recvQueue: seq[byte]
     isReset: bool
+    remoteReset: bool
     closedRemotely: Future[void]
     closedLocally: bool
     receivedData: AsyncEvent
@@ -194,9 +195,11 @@ method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
     await channel.actuallyClose()
 
 proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
-  if not channel.isReset:
+  if channel.isReset:
+    return
   trace "Reset channel"
   channel.isReset = true
+  channel.remoteReset = not isLocal
   for (d, s, fut) in channel.sendQueue:
     fut.fail(newLPStreamEOFError())
   channel.sendQueue = @[]
@@ -235,7 +238,15 @@ method readOnce*(
   nbytes: int):
   Future[int] {.async.} =
 
-  if channel.returnedEof: raise newLPStreamEOFError()
+  if channel.isReset:
+    raise if channel.remoteReset:
+        newLPStreamResetError()
+      elif channel.closedLocally:
+        newLPStreamClosedError()
+      else:
+        newLPStreamConnDownError()
+  if channel.returnedEof:
+    raise newLPStreamRemoteClosedError()
   if channel.recvQueue.len == 0:
     channel.receivedData.clear()
     await channel.closedRemotely or channel.receivedData.wait()
@@ -313,8 +324,9 @@ proc trySend(channel: YamuxChannel) {.async.} =
     channel.sendWindow.dec(toSend)
     try: await channel.conn.write(sendBuffer)
     except CatchableError as exc:
+      let connDown = newLPStreamConnDownError(exc)
       for fut in futures.items():
-        fut.fail(exc)
+        fut.fail(connDown)
       await channel.reset()
       break
     for fut in futures.items():
@@ -323,8 +335,11 @@ proc trySend(channel: YamuxChannel) {.async.} =
 
 method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
   result = newFuture[void]("Yamux Send")
+  if channel.remoteReset:
+    result.fail(newLPStreamResetError())
+    return result
   if channel.closedLocally or channel.isReset:
-    result.fail(newLPStreamEOFError())
+    result.fail(newLPStreamClosedError())
     return result
   if msg.len == 0:
     result.complete()
@@ -396,8 +411,9 @@ method close*(m: Yamux) {.async.} =
   m.isClosed = true
 
   trace "Closing yamux"
-  for channel in m.channels.values:
+  let channels = toSeq(m.channels.values())
-    await channel.reset()
+  for channel in channels:
+    await channel.reset(true)
   await m.connection.write(YamuxHeader.goAway(NormalTermination))
   await m.connection.close()
   trace "Closed yamux"
@@ -453,6 +469,7 @@ method handle*(m: Yamux) {.async, gcsafe.} =
             m.flushed[header.streamId].dec(int(header.length))
             if m.flushed[header.streamId] < 0:
               raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
+            if header.length > 0:
               var buffer = newSeqUninitialized[byte](header.length)
               await m.connection.readExactly(addr buffer[0], int(header.length))
           continue

libp2p/nameresolving/dnsresolver.nim

@@ -14,7 +14,7 @@ else:
 
 import
   std/[streams, strutils, sets, sequtils],
-  chronos, chronicles,
+  chronos, chronicles, stew/byteutils,
   dnsclientpkg/[protocol, types]
 
 import
@@ -76,15 +76,11 @@ proc getDnsResponse(
     if not receivedDataFuture.finished:
       raise newException(IOError, "DNS server timeout")
 
-    var
+    let rawResponse = sock.getMessage()
-      rawResponse = sock.getMessage()
-      dataStream = newStringStream()
-    dataStream.writeData(addr rawResponse[0], rawResponse.len)
-    dataStream.setPosition(0)
     # parseResponse can has a raises: [Exception, ..] because of
     # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
     # it can't actually raise though
-    return parseResponse(dataStream)
+    return parseResponse(string.fromBytes(rawResponse))
   except CatchableError as exc: raise exc
   except Exception as exc: raiseAssert exc.msg
   finally:
@@ -118,7 +114,14 @@ method resolveIp*(
       try:
         let resp = await fut
         for answer in resp.answers:
-          resolvedAddresses.incl(answer.toString())
+          # toString can has a raises: [Exception, ..] because of
+          # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
+          # it can't actually raise though
+          resolvedAddresses.incl(
+            try: answer.toString()
+            except CatchableError as exc: raise exc
+            except Exception as exc: raiseAssert exc.msg
+          )
       except CancelledError as e:
         raise e
       except ValueError as e:
@@ -158,6 +161,11 @@ method resolveTxt*(
       self.nameServers.add(self.nameServers[0])
       self.nameServers.delete(0)
       continue
+    except Exception as e:
+      # toString can has a raises: [Exception, ..] because of
+      # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
+      # it can't actually raise though
+      raiseAssert e.msg
 
   debug "Failed to resolve TXT, returning empty set"
   return @[]

libp2p/nameresolving/nameresolver.nim

@@ -44,11 +44,13 @@ method resolveIp*(
   doAssert(false, "Not implemented!")
 
 proc getHostname*(ma: MultiAddress): string =
-  let firstPart = ($ma[0].get()).split('/')
+  let
-  if firstPart.len > 1: firstPart[2]
+    firstPart = ma[0].valueOr: return ""
+    fpSplitted = ($firstPart).split('/', 2)
+  if fpSplitted.len > 2: fpSplitted[2]
   else: ""
 
-proc resolveDnsAddress(
+proc resolveOneAddress(
   self: NameResolver,
   ma: MultiAddress,
   domain: Domain = Domain.AF_UNSPEC,
@@ -69,24 +71,17 @@ proc resolveDnsAddress(
     for address in resolvedAddresses:
       var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet()
       for part in ma:
-        if DNS.match(part.get()): continue
+        if DNS.match(part.tryGet()): continue
         createdAddress &= part.tryGet()
       createdAddress
 
-func matchDnsSuffix(m1, m2: MultiAddress): MaResult[bool] =
+proc resolveDnsAddr*(
-  for partMaybe in m1:
-    let part = ?partMaybe
-    if DNS.match(part): continue
-    let entryProt = ?m2[?part.protoCode()]
-    if entryProt != part:
-      return ok(false)
-  return ok(true)
-
-proc resolveDnsAddr(
   self: NameResolver,
   ma: MultiAddress,
-  depth: int = 0): Future[seq[MultiAddress]]
+  depth: int = 0): Future[seq[MultiAddress]] {.async.} =
-  {.async.} =
+
+  if not DNSADDR.matchPartial(ma):
+    return @[ma]
 
   trace "Resolving dnsaddr", ma
   if depth > 6:
@@ -104,21 +99,17 @@ proc resolveDnsAddr(
     if not entry.startsWith("dnsaddr="): continue
     let entryValue = MultiAddress.init(entry[8..^1]).tryGet()
 
-    if not matchDnsSuffix(ma, entryValue).tryGet(): continue
+    if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet():
+      if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]:
+        continue
 
-    # The spec is not clear wheter only DNSADDR can be recursived
-    # or any DNS addr. Only handling DNSADDR because it's simpler
-    # to avoid infinite recursion
-    if DNSADDR.matchPartial(entryValue):
     let resolved = await self.resolveDnsAddr(entryValue, depth + 1)
     for r in resolved:
       result.add(r)
-    else:
-      result.add(entryValue)
 
   if result.len == 0:
-    debug "Failed to resolve any DNSADDR", ma
+    debug "Failed to resolve a DNSADDR", ma
-    return @[ma]
+    return @[]
   return result
 
 
@@ -133,14 +124,15 @@ proc resolveMAddress*(
     let code = address[0].get().protoCode().get()
     let seq = case code:
       of multiCodec("dns"):
-        await self.resolveDnsAddress(address)
+        await self.resolveOneAddress(address)
       of multiCodec("dns4"):
-        await self.resolveDnsAddress(address, Domain.AF_INET)
+        await self.resolveOneAddress(address, Domain.AF_INET)
       of multiCodec("dns6"):
-        await self.resolveDnsAddress(address, Domain.AF_INET6)
+        await self.resolveOneAddress(address, Domain.AF_INET6)
       of multiCodec("dnsaddr"):
         await self.resolveDnsAddr(address)
       else:
+        doAssert false
         @[address]
     for ad in seq:
       res.incl(ad)

libp2p/peerid.nim

@@ -148,7 +148,7 @@ func init*(pid: var PeerId, data: string): bool =
   if Base58.decode(data, p, length) == Base58Status.Success:
     p.setLen(length)
     var opid: PeerId
-    shallowCopy(opid.data, p)
+    opid.data = p
     if opid.validate():
       pid = opid
       result = true

libp2p/peerinfo.nim

@@ -22,11 +22,17 @@ export peerid, multiaddress, crypto, routing_record, errors, results
 ## Our local peer info
 
 type
-  PeerInfoError* = LPError
+  PeerInfoError* = object of LPError
+
+  AddressMapper* =
+    proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]]
+      {.gcsafe, raises: [Defect].}
 
   PeerInfo* {.public.} = ref object
     peerId*: PeerId
-    addrs*: seq[MultiAddress]
+    listenAddrs*: seq[MultiAddress]
+    addrs: seq[MultiAddress]
+    addressMappers*: seq[AddressMapper]
     protocols*: seq[string]
     protoVersion*: string
     agentVersion*: string
@@ -37,6 +43,7 @@ type
 func shortLog*(p: PeerInfo): auto =
   (
     peerId: $p.peerId,
+    listenAddrs: mapIt(p.listenAddrs, $it),
     addrs: mapIt(p.addrs, $it),
     protocols: mapIt(p.protocols, $it),
     protoVersion: p.protoVersion,
@@ -44,7 +51,11 @@ func shortLog*(p: PeerInfo): auto =
   )
 chronicles.formatIt(PeerInfo): shortLog(it)
 
-proc update*(p: PeerInfo) =
+proc update*(p: PeerInfo) {.async.} =
+  p.addrs = p.listenAddrs
+  for mapper in p.addressMappers:
+    p.addrs = await mapper(p.addrs)
+
   let sprRes = SignedPeerRecord.init(
     p.privateKey,
     PeerRecord.init(p.peerId, p.addrs)
@@ -55,14 +66,19 @@ proc update*(p: PeerInfo) =
     discard
     #info "Can't update the signed peer record"
 
+proc addrs*(p: PeerInfo): seq[MultiAddress] =
+  p.addrs
+
 proc new*(
   p: typedesc[PeerInfo],
   key: PrivateKey,
-  addrs: openArray[MultiAddress] = [],
+  listenAddrs: openArray[MultiAddress] = [],
   protocols: openArray[string] = [],
   protoVersion: string = "",
-  agentVersion: string = ""): PeerInfo
+  agentVersion: string = "",
-  {.raises: [Defect, PeerInfoError].} =
+  addressMappers = newSeq[AddressMapper](),
+  ): PeerInfo
+  {.raises: [Defect, LPError].} =
 
   let pubkey = try:
       key.getPublicKey().tryGet()
@@ -77,10 +93,9 @@ proc new*(
     privateKey: key,
     protoVersion: protoVersion,
     agentVersion: agentVersion,
-    addrs: @addrs,
+    listenAddrs: @listenAddrs,
     protocols: @protocols,
+    addressMappers: addressMappers
   )
 
-  peerInfo.update()
-
   return peerInfo

libp2p/protobuf/minprotobuf.nim

@@ -124,7 +124,7 @@ proc vsizeof*(field: ProtoField): int {.inline.} =
 proc initProtoBuffer*(data: seq[byte], offset = 0,
                       options: set[ProtoFlags] = {}): ProtoBuffer =
   ## Initialize ProtoBuffer with shallow copy of ``data``.
-  shallowCopy(result.buffer, data)
+  result.buffer = data
   result.offset = offset
   result.options = options
 

libp2p/protocols/connectivity/autonat.nim

@@ -13,14 +13,15 @@ else:
   {.push raises: [].}
 
 import std/[options, sets, sequtils]
+import stew/results
 import chronos, chronicles, stew/objects
-import ./protocol,
+import ../protocol,
-       ../switch,
+       ../../switch,
-       ../multiaddress,
+       ../../multiaddress,
-       ../multicodec,
+       ../../multicodec,
-       ../peerid,
+       ../../peerid,
-       ../utils/semaphore,
+       ../../utils/semaphore,
-       ../errors
+       ../../errors
 
 logScope:
   topics = "libp2p autonat"
@@ -226,7 +227,10 @@ proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} =
   try:
     await a.sem.acquire()
     let ma = await a.switch.dialer.tryDial(conn.peerId, addrs)
-    await conn.sendResponseOk(ma)
+    if ma.isSome:
+      await conn.sendResponseOk(ma.get())
+    else:
+      await conn.sendResponseError(DialError, "Missing observed address")
   except CancelledError as exc:
     raise exc
   except CatchableError as exc:
@@ -241,15 +245,19 @@ proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] =
   if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId:
     return conn.sendResponseError(BadRequest, "PeerId mismatch")
 
-  var isRelayed = conn.observedAddr.contains(multiCodec("p2p-circuit"))
+  if conn.observedAddr.isNone:
+    return conn.sendResponseError(BadRequest, "Missing observed address")
+  let observedAddr = conn.observedAddr.get()
+
+  var isRelayed = observedAddr.contains(multiCodec("p2p-circuit"))
   if isRelayed.isErr() or isRelayed.get():
     return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address")
-  let hostIp = conn.observedAddr[0]
+  let hostIp = observedAddr[0]
   if hostIp.isErr() or not IP.match(hostIp.get()):
-    trace "wrong observed address", address=conn.observedAddr
+    trace "wrong observed address", address=observedAddr
     return conn.sendResponseError(InternalError, "Expected an IP address")
   var addrs = initHashSet[MultiAddress]()
-  addrs.incl(conn.observedAddr)
+  addrs.incl(observedAddr)
   for ma in peerInfo.addrs:
     isRelayed = ma.contains(multiCodec("p2p-circuit"))
     if isRelayed.isErr() or isRelayed.get():

libp2p/protocols/connectivity/relay/client.nim

@@ -20,10 +20,10 @@ import ./relay,
        ./messages,
        ./rconn,
        ./utils,
-       ../../peerinfo,
+       ../../../peerinfo,
-       ../../switch,
+       ../../../switch,
-       ../../multiaddress,
+       ../../../multiaddress,
-       ../../stream/connection
+       ../../../stream/connection
 
 
 logScope:

libp2p/protocols/connectivity/relay/messages.nim

@@ -14,8 +14,8 @@ else:
 
 import options, macros, sequtils
 import stew/objects
-import ../../peerinfo,
+import ../../../peerinfo,
-       ../../signed_envelope
+       ../../../signed_envelope
 
 # Circuit Relay V1 Message
 

libp2p/protocols/connectivity/relay/rconn.nim

@@ -14,7 +14,7 @@ else:
 
 import chronos
 
-import ../../stream/connection
+import ../../../stream/connection
 
 type
   RelayConnection* = ref object of Connection

libp2p/protocols/connectivity/relay/relay.nim

@@ -19,16 +19,16 @@ import chronos, chronicles
 import ./messages,
        ./rconn,
        ./utils,
-       ../../peerinfo,
+       ../../../peerinfo,
-       ../../switch,
+       ../../../switch,
-       ../../multiaddress,
+       ../../../multiaddress,
-       ../../multicodec,
+       ../../../multicodec,
-       ../../stream/connection,
+       ../../../stream/connection,
-       ../../protocols/protocol,
+       ../../../protocols/protocol,
-       ../../transports/transport,
+       ../../../transports/transport,
-       ../../errors,
+       ../../../errors,
-       ../../utils/heartbeat,
+       ../../../utils/heartbeat,
-       ../../signed_envelope
+       ../../../signed_envelope
 
 # TODO:
 # * Eventually replace std/times by chronos/timer. Currently chronos/timer

libp2p/protocols/connectivity/relay/rtransport.nim

@@ -19,9 +19,9 @@ import chronos, chronicles
 import ./client,
        ./rconn,
        ./utils,
-       ../../switch,
+       ../../../switch,
-       ../../stream/connection,
+       ../../../stream/connection,
-       ../../transports/transport
+       ../../../transports/transport
 
 logScope:
   topics = "libp2p relay relay-transport"

libp2p/protocols/connectivity/relay/utils.nim

@@ -17,7 +17,7 @@ import options
 import chronos, chronicles
 
 import ./messages,
-       ../../stream/connection
+       ../../../stream/connection
 
 logScope:
   topics = "libp2p relay relay-utils"
@@ -64,12 +64,14 @@ proc bridge*(connSrc: Connection, connDst: Connection) {.async.} =
       await futSrc or futDst
       if futSrc.finished():
         bufRead = await futSrc
+        if bufRead > 0:
           bytesSendFromSrcToDst.inc(bufRead)
           await connDst.write(@bufSrcToDst[0..<bufRead])
           zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
         futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
       if futDst.finished():
         bufRead = await futDst
+        if bufRead > 0:
           bytesSendFromDstToSrc += bufRead
           await connSrc.write(bufDstToSrc[0..<bufRead])
           zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)

libp2p/protocols/identify.nim

@@ -16,6 +16,7 @@ else:
   {.push raises: [].}
 
 import std/[sequtils, options, strutils, sugar]
+import stew/results
 import chronos, chronicles
 import ../protobuf/minprotobuf,
        ../peerinfo,
@@ -80,7 +81,7 @@ chronicles.expandIt(IdentifyInfo):
     if iinfo.signedPeerRecord.isSome(): "Some"
     else: "None"
 
-proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): ProtoBuffer
+proc encodeMsg(peerInfo: PeerInfo, observedAddr: Opt[MultiAddress], sendSpr: bool): ProtoBuffer
   {.raises: [Defect].} =
   result = initProtoBuffer()
 
@@ -91,7 +92,8 @@ proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): P
     result.write(2, ma.data.buffer)
   for proto in peerInfo.protocols:
     result.write(3, proto)
-  result.write(4, observedAddr.data.buffer)
+  if observedAddr.isSome:
+    result.write(4, observedAddr.get().data.buffer)
   let protoVersion = ProtoVersion
   result.write(5, protoVersion)
   let agentVersion = if peerInfo.agentVersion.len <= 0:

libp2p/protocols/pubsub/gossipsub/types.nim

@@ -16,7 +16,7 @@ import chronos
 import std/[tables, sets]
 import ".."/[floodsub, peertable, mcache, pubsubpeer]
 import "../rpc"/[messages]
-import "../../.."/[peerid, multiaddress]
+import "../../.."/[peerid, multiaddress, utility]
 
 const
   GossipSubCodec* = "/meshsub/1.1.0"
@@ -65,7 +65,7 @@ type
     meshFailurePenalty*: float64
     invalidMessageDeliveries*: float64
 
-  TopicParams* = object
+  TopicParams* {.public.} = object
     topicWeight*: float64
 
     # p1
@@ -102,7 +102,7 @@ type
     appScore*: float64 # application specific score
     behaviourPenalty*: float64 # the eventual penalty score
 
-  GossipSubParams* = object
+  GossipSubParams* {.public.} = object
     explicit*: bool
     pruneBackoff*: Duration
     unsubscribeBackoff*: Duration

libp2p/protocols/pubsub/pubsub.nim

@@ -292,19 +292,11 @@ proc getOrCreatePeer*(
   proc getConn(): Future[Connection] {.async.} =
     return await p.switch.dial(peerId, protos)
 
-  proc dropConn(peer: PubSubPeer) =
-    proc dropConnAsync(peer: PubSubPeer) {.async.} =
-      try:
-        await p.switch.disconnect(peer.peerId)
-      except CatchableError as exc: # never cancelled
-        trace "Failed to close connection", peer, error = exc.name, msg = exc.msg
-    asyncSpawn dropConnAsync(peer)
-
   proc onEvent(peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe.} =
     p.onPubSubPeerEvent(peer, event)
 
   # create new pubsub peer
-  let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, onEvent, protos[0], p.maxMessageSize)
+  let pubSubPeer = PubSubPeer.new(peerId, getConn, onEvent, protos[0], p.maxMessageSize)
   debug "created new pubsub peer", peerId
 
   p.peers[peerId] = pubSubPeer

libp2p/protocols/pubsub/pubsubpeer.nim

@@ -12,7 +12,8 @@ when (NimMajor, NimMinor) < (1, 4):
 else:
   {.push raises: [].}
 
-import std/[sequtils, strutils, tables, hashes]
+import std/[sequtils, strutils, tables, hashes, options]
+import stew/results
 import chronos, chronicles, nimcrypto/sha2, metrics
 import rpc/[messages, message, protobuf],
        ../../peerid,
@@ -51,7 +52,6 @@ type
 
   PubSubPeer* = ref object of RootObj
     getConn*: GetConn                   # callback to establish a new send connection
-    dropConn*: DropConn                 # Function pointer to use to drop connections
     onEvent*: OnEvent                   # Connectivity updates for peer
     codec*: string                      # the protocol that this peer joined from
     sendConn*: Connection               # cached send connection
@@ -175,7 +175,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} =
 
     trace "Get new send connection", p, newConn
     p.sendConn = newConn
-    p.address = some(p.sendConn.observedAddr)
+    p.address = if p.sendConn.observedAddr.isSome: some(p.sendConn.observedAddr.get) else: none(MultiAddress)
 
     if p.onEvent != nil:
       p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Connected))
@@ -206,9 +206,6 @@ proc connectImpl(p: PubSubPeer) {.async.} =
       await connectOnce(p)
   except CatchableError as exc: # never cancelled
     debug "Could not establish send connection", msg = exc.msg
-  finally:
-    # drop the connection, else we end up with ghost peers
-    if p.dropConn != nil: p.dropConn(p)
 
 proc connect*(p: PubSubPeer) =
   asyncSpawn connectImpl(p)
@@ -286,14 +283,12 @@ proc new*(
   T: typedesc[PubSubPeer],
   peerId: PeerId,
   getConn: GetConn,
-  dropConn: DropConn,
   onEvent: OnEvent,
   codec: string,
   maxMessageSize: int): T =
 
   T(
     getConn: getConn,
-    dropConn: dropConn,
     onEvent: onEvent,
     codec: codec,
     peerId: peerId,

libp2p/protocols/rendezvous.nim

@@ -0,0 +1,675 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import tables, sequtils, sugar, sets, options
+import chronos,
+       chronicles,
+       bearssl/rand,
+       stew/[byteutils, objects]
+import ./protocol,
+       ../switch,
+       ../routing_record,
+       ../utils/heartbeat,
+       ../stream/connection,
+       ../utils/offsettedseq,
+       ../utils/semaphore
+
+export chronicles
+
+logScope:
+  topics = "libp2p discovery rendezvous"
+
+const
+  RendezVousCodec* = "/rendezvous/1.0.0"
+  MinimumDuration* = 2.hours
+  MaximumDuration = 72.hours
+  MinimumTTL = MinimumDuration.seconds.uint64
+  MaximumTTL = MaximumDuration.seconds.uint64
+  RegistrationLimitPerPeer = 1000
+  DiscoverLimit = 1000'u64
+  SemaphoreDefaultSize = 5
+
+type
+  MessageType {.pure.} = enum
+    Register = 0
+    RegisterResponse = 1
+    Unregister = 2
+    Discover = 3
+    DiscoverResponse = 4
+
+  ResponseStatus = enum
+    Ok = 0
+    InvalidNamespace = 100
+    InvalidSignedPeerRecord = 101
+    InvalidTTL = 102
+    InvalidCookie = 103
+    NotAuthorized = 200
+    InternalError = 300
+    Unavailable = 400
+
+  Cookie = object
+    offset : uint64
+    ns : string
+
+  Register = object
+    ns : string
+    signedPeerRecord: seq[byte]
+    ttl: Option[uint64] # in seconds
+
+  RegisterResponse = object
+    status: ResponseStatus
+    text: Option[string]
+    ttl: Option[uint64] # in seconds
+
+  Unregister = object
+    ns: string
+
+  Discover = object
+    ns: string
+    limit: Option[uint64]
+    cookie: Option[seq[byte]]
+
+  DiscoverResponse = object
+    registrations: seq[Register]
+    cookie: Option[seq[byte]]
+    status: ResponseStatus
+    text: Option[string]
+
+  Message = object
+    msgType: MessageType
+    register: Option[Register]
+    registerResponse: Option[RegisterResponse]
+    unregister: Option[Unregister]
+    discover: Option[Discover]
+    discoverResponse: Option[DiscoverResponse]
+
+proc encode(c: Cookie): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, c.offset)
+  result.write(2, c.ns)
+  result.finish()
+
+proc encode(r: Register): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, r.ns)
+  result.write(2, r.signedPeerRecord)
+  if r.ttl.isSome():
+    result.write(3, r.ttl.get())
+  result.finish()
+
+proc encode(rr: RegisterResponse): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, rr.status.uint)
+  if rr.text.isSome():
+    result.write(2, rr.text.get())
+  if rr.ttl.isSome():
+    result.write(3, rr.ttl.get())
+  result.finish()
+
+proc encode(u: Unregister): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, u.ns)
+  result.finish()
+
+proc encode(d: Discover): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, d.ns)
+  if d.limit.isSome():
+    result.write(2, d.limit.get())
+  if d.cookie.isSome():
+    result.write(3, d.cookie.get())
+  result.finish()
+
+proc encode(d: DiscoverResponse): ProtoBuffer =
+  result = initProtoBuffer()
+  for reg in d.registrations:
+    result.write(1, reg.encode())
+  if d.cookie.isSome():
+    result.write(2, d.cookie.get())
+  result.write(3, d.status.uint)
+  if d.text.isSome():
+    result.write(4, d.text.get())
+  result.finish()
+
+proc encode(msg: Message): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, msg.msgType.uint)
+  if msg.register.isSome():
+    result.write(2, msg.register.get().encode())
+  if msg.registerResponse.isSome():
+    result.write(3, msg.registerResponse.get().encode())
+  if msg.unregister.isSome():
+    result.write(4, msg.unregister.get().encode())
+  if msg.discover.isSome():
+    result.write(5, msg.discover.get().encode())
+  if msg.discoverResponse.isSome():
+    result.write(6, msg.discoverResponse.get().encode())
+  result.finish()
+
+proc decode(_: typedesc[Cookie], buf: seq[byte]): Option[Cookie] =
+  var c: Cookie
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, c.offset)
+    r2 = pb.getRequiredField(2, c.ns)
+  if r1.isErr() or r2.isErr(): return none(Cookie)
+  some(c)
+
+proc decode(_: typedesc[Register], buf: seq[byte]): Option[Register] =
+  var
+    r: Register
+    ttl: uint64
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, r.ns)
+    r2 = pb.getRequiredField(2, r.signedPeerRecord)
+    r3 = pb.getField(3, ttl)
+  if r1.isErr() or r2.isErr() or r3.isErr(): return none(Register)
+  if r3.get(): r.ttl = some(ttl)
+  some(r)
+
+proc decode(_: typedesc[RegisterResponse], buf: seq[byte]): Option[RegisterResponse] =
+  var
+    rr: RegisterResponse
+    statusOrd: uint
+    text: string
+    ttl: uint64
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, statusOrd)
+    r2 = pb.getField(2, text)
+    r3 = pb.getField(3, ttl)
+  if r1.isErr() or r2.isErr() or r3.isErr() or
+     not checkedEnumAssign(rr.status, statusOrd): return none(RegisterResponse)
+  if r2.get(): rr.text = some(text)
+  if r3.get(): rr.ttl = some(ttl)
+  some(rr)
+
+proc decode(_: typedesc[Unregister], buf: seq[byte]): Option[Unregister] =
+  var u: Unregister
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, u.ns)
+  if r1.isErr(): return none(Unregister)
+  some(u)
+
+proc decode(_: typedesc[Discover], buf: seq[byte]): Option[Discover] =
+  var
+    d: Discover
+    limit: uint64
+    cookie: seq[byte]
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, d.ns)
+    r2 = pb.getField(2, limit)
+    r3 = pb.getField(3, cookie)
+  if r1.isErr() or r2.isErr() or r3.isErr: return none(Discover)
+  if r2.get(): d.limit = some(limit)
+  if r3.get(): d.cookie = some(cookie)
+  some(d)
+
+proc decode(_: typedesc[DiscoverResponse], buf: seq[byte]): Option[DiscoverResponse] =
+  var
+    dr: DiscoverResponse
+    registrations: seq[seq[byte]]
+    cookie: seq[byte]
+    statusOrd: uint
+    text: string
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRepeatedField(1, registrations)
+    r2 = pb.getField(2, cookie)
+    r3 = pb.getRequiredField(3, statusOrd)
+    r4 = pb.getField(4, text)
+  if r1.isErr() or r2.isErr() or r3.isErr or r4.isErr() or
+     not checkedEnumAssign(dr.status, statusOrd): return none(DiscoverResponse)
+  for reg in registrations:
+    var r: Register
+    let regOpt = Register.decode(reg)
+    if regOpt.isNone(): return none(DiscoverResponse)
+    dr.registrations.add(regOpt.get())
+  if r2.get(): dr.cookie = some(cookie)
+  if r4.get(): dr.text = some(text)
+  some(dr)
+
+proc decode(_: typedesc[Message], buf: seq[byte]): Option[Message] =
+  var
+    msg: Message
+    statusOrd: uint
+    pbr, pbrr, pbu, pbd, pbdr: ProtoBuffer
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, statusOrd)
+    r2 = pb.getField(2, pbr)
+    r3 = pb.getField(3, pbrr)
+    r4 = pb.getField(4, pbu)
+    r5 = pb.getField(5, pbd)
+    r6 = pb.getField(6, pbdr)
+  if r1.isErr() or r2.isErr() or r3.isErr() or
+     r4.isErr() or r5.isErr() or r6.isErr() or
+     not checkedEnumAssign(msg.msgType, statusOrd): return none(Message)
+  if r2.get():
+    msg.register = Register.decode(pbr.buffer)
+    if msg.register.isNone(): return none(Message)
+  if r3.get():
+    msg.registerResponse = RegisterResponse.decode(pbrr.buffer)
+    if msg.registerResponse.isNone(): return none(Message)
+  if r4.get():
+    msg.unregister = Unregister.decode(pbu.buffer)
+    if msg.unregister.isNone(): return none(Message)
+  if r5.get():
+    msg.discover = Discover.decode(pbd.buffer)
+    if msg.discover.isNone(): return none(Message)
+  if r6.get():
+    msg.discoverResponse = DiscoverResponse.decode(pbdr.buffer)
+    if msg.discoverResponse.isNone(): return none(Message)
+  some(msg)
+
+
+type
+  RendezVousError* = object of LPError
+  RegisteredData = object
+    expiration: Moment
+    peerId: PeerId
+    data: Register
+
+  RendezVous* = ref object of LPProtocol
+    # Registered needs to be an offsetted sequence
+    # because we need stable index for the cookies.
+    registered: OffsettedSeq[RegisteredData]
+    # Namespaces is a table whose key is a salted namespace and
+    # the value is the index sequence corresponding to this
+    # namespace in the offsettedqueue.
+    namespaces: Table[string, seq[int]]
+    rng: ref HmacDrbgContext
+    salt: string
+    defaultDT: Moment
+    registerDeletionLoop: Future[void]
+    #registerEvent: AsyncEvent # TODO: to raise during the heartbeat
+    # + make the heartbeat sleep duration "smarter"
+    sema: AsyncSemaphore
+    peers: seq[PeerId]
+    cookiesSaved: Table[PeerId, Table[string, seq[byte]]]
+    switch: Switch
+
+proc checkPeerRecord(spr: seq[byte], peerId: PeerId): Result[void, string] =
+  if spr.len == 0: return err("Empty peer record")
+  let signedEnv = ? SignedPeerRecord.decode(spr).mapErr(x => $x)
+  if signedEnv.data.peerId != peerId:
+    return err("Bad Peer ID")
+  return ok()
+
+proc sendRegisterResponse(conn: Connection,
+                          ttl: uint64) {.async.} =
+  let msg = encode(Message(
+        msgType: MessageType.RegisterResponse,
+        registerResponse: some(RegisterResponse(status: Ok, ttl: some(ttl)))))
+  await conn.writeLp(msg.buffer)
+
+proc sendRegisterResponseError(conn: Connection,
+                               status: ResponseStatus,
+                               text: string = "") {.async.} =
+  let msg = encode(Message(
+        msgType: MessageType.RegisterResponse,
+        registerResponse: some(RegisterResponse(status: status, text: some(text)))))
+  await conn.writeLp(msg.buffer)
+
+proc sendDiscoverResponse(conn: Connection,
+                          s: seq[Register],
+                          cookie: Cookie) {.async.} =
+  let msg = encode(Message(
+        msgType: MessageType.DiscoverResponse,
+        discoverResponse: some(DiscoverResponse(
+            status: Ok,
+            registrations: s,
+            cookie: some(cookie.encode().buffer)
+          ))
+        ))
+  await conn.writeLp(msg.buffer)
+
+proc sendDiscoverResponseError(conn: Connection,
+                               status: ResponseStatus,
+                               text: string = "") {.async.} =
+  let msg = encode(Message(
+        msgType: MessageType.DiscoverResponse,
+        discoverResponse: some(DiscoverResponse(status: status, text: some(text)))))
+  await conn.writeLp(msg.buffer)
+
+proc countRegister(rdv: RendezVous, peerId: PeerId): int =
+  let n = Moment.now()
+  for data in rdv.registered:
+    if data.peerId == peerId and data.expiration > n:
+      result.inc()
+
+proc save(rdv: RendezVous,
+          ns: string,
+          peerId: PeerId,
+          r: Register,
+          update: bool = true) =
+  let nsSalted = ns & rdv.salt
+  discard rdv.namespaces.hasKeyOrPut(nsSalted, newSeq[int]())
+  try:
+    for index in rdv.namespaces[nsSalted]:
+      if rdv.registered[index].peerId == peerId:
+        if update == false: return
+        rdv.registered[index].expiration = rdv.defaultDT
+    rdv.registered.add(
+      RegisteredData(
+        peerId: peerId,
+        expiration: Moment.now() + r.ttl.get(MinimumTTL).int64.seconds,
+        data: r
+      )
+    )
+    rdv.namespaces[nsSalted].add(rdv.registered.high)
+#    rdv.registerEvent.fire()
+  except KeyError:
+    doAssert false, "Should have key"
+
+proc register(rdv: RendezVous, conn: Connection, r: Register): Future[void] =
+  trace "Received Register", peerId = conn.peerId, ns = r.ns
+  if r.ns.len notin 1..255:
+    return conn.sendRegisterResponseError(InvalidNamespace)
+  let ttl = r.ttl.get(MinimumTTL)
+  if ttl notin MinimumTTL..MaximumTTL:
+    return conn.sendRegisterResponseError(InvalidTTL)
+  let pr = checkPeerRecord(r.signedPeerRecord, conn.peerId)
+  if pr.isErr():
+    return conn.sendRegisterResponseError(InvalidSignedPeerRecord, pr.error())
+  if rdv.countRegister(conn.peerId) >= RegistrationLimitPerPeer:
+    return conn.sendRegisterResponseError(NotAuthorized, "Registration limit reached")
+  rdv.save(r.ns, conn.peerId, r)
+  conn.sendRegisterResponse(ttl)
+
+proc unregister(rdv: RendezVous, conn: Connection, u: Unregister) =
+  trace "Received Unregister", peerId = conn.peerId, ns = u.ns
+  let nsSalted = u.ns & rdv.salt
+  try:
+    for index in rdv.namespaces[nsSalted]:
+      if rdv.registered[index].peerId == conn.peerId:
+        rdv.registered[index].expiration = rdv.defaultDT
+  except KeyError:
+    return
+
+proc discover(rdv: RendezVous, conn: Connection, d: Discover) {.async.} =
+  trace "Received Discover", peerId = conn.peerId, ns = d.ns
+  if d.ns.len notin 0..255:
+    await conn.sendDiscoverResponseError(InvalidNamespace)
+    return
+  var limit = min(DiscoverLimit, d.limit.get(DiscoverLimit))
+  var
+    cookie =
+      if d.cookie.isSome():
+        try:
+          Cookie.decode(d.cookie.get()).get()
+        except CatchableError:
+          await conn.sendDiscoverResponseError(InvalidCookie)
+          return
+      else: Cookie(offset: rdv.registered.low().uint64 - 1)
+  if cookie.ns != d.ns or
+      cookie.offset notin rdv.registered.low().uint64..rdv.registered.high().uint64:
+    cookie = Cookie(offset: rdv.registered.low().uint64 - 1)
+  let
+    nsSalted = d.ns & rdv.salt
+    namespaces =
+      if d.ns != "":
+        try:
+          rdv.namespaces[nsSalted]
+        except KeyError:
+          await conn.sendDiscoverResponseError(InvalidNamespace)
+          return
+      else: toSeq(cookie.offset.int..rdv.registered.high())
+  if namespaces.len() == 0:
+    await conn.sendDiscoverResponse(@[], Cookie())
+    return
+  var offset = namespaces[^1]
+  let n = Moment.now()
+  var s = collect(newSeq()):
+      for index in namespaces:
+        var reg = rdv.registered[index]
+        if limit == 0:
+          offset = index
+          break
+        if reg.expiration < n or index.uint64 <= cookie.offset: continue
+        limit.dec()
+        reg.data.ttl = some((reg.expiration - Moment.now()).seconds.uint64)
+        reg.data
+  rdv.rng.shuffle(s)
+  await conn.sendDiscoverResponse(s, Cookie(offset: offset.uint64, ns: d.ns))
+
+proc advertisePeer(rdv: RendezVous,
+                   peer: PeerId,
+                   msg: seq[byte]) {.async.} =
+  proc advertiseWrap() {.async.} =
+    try:
+      let conn = await rdv.switch.dial(peer, RendezVousCodec)
+      defer: await conn.close()
+      await conn.writeLp(msg)
+      let
+        buf = await conn.readLp(4096)
+        msgRecv = Message.decode(buf).get()
+      if msgRecv.msgType != MessageType.RegisterResponse:
+        trace "Unexpected register response", peer, msgType = msgRecv.msgType
+      elif msgRecv.registerResponse.isNone() or
+           msgRecv.registerResponse.get().status != ResponseStatus.Ok:
+        trace "Refuse to register", peer, response = msgRecv.registerResponse
+    except CatchableError as exc:
+      trace "exception in the advertise", error = exc.msg
+    finally:
+      rdv.sema.release()
+  await rdv.sema.acquire()
+  discard await advertiseWrap().withTimeout(5.seconds)
+
+proc advertise*(rdv: RendezVous,
+                ns: string,
+                ttl: Duration = MinimumDuration) {.async.} =
+  let sprBuff = rdv.switch.peerInfo.signedPeerRecord.encode()
+  if sprBuff.isErr():
+    raise newException(RendezVousError, "Wrong Signed Peer Record")
+  if ns.len notin 1..255:
+    raise newException(RendezVousError, "Invalid namespace")
+  if ttl notin MinimumDuration..MaximumDuration:
+    raise newException(RendezVousError, "Invalid time to live")
+  let
+    r = Register(ns: ns, signedPeerRecord: sprBuff.get(), ttl: some(ttl.seconds.uint64))
+    msg = encode(Message(msgType: MessageType.Register, register: some(r)))
+  rdv.save(ns, rdv.switch.peerInfo.peerId, r)
+  let fut = collect(newSeq()):
+    for peer in rdv.peers:
+      trace "Send Advertise", peerId = peer, ns
+      rdv.advertisePeer(peer, msg.buffer)
+  await allFutures(fut)
+
+proc requestLocally*(rdv: RendezVous, ns: string): seq[PeerRecord] =
+  let
+    nsSalted = ns & rdv.salt
+    n = Moment.now()
+  try:
+    collect(newSeq()):
+      for index in rdv.namespaces[nsSalted]:
+        if rdv.registered[index].expiration > n:
+          SignedPeerRecord.decode(rdv.registered[index].data.signedPeerRecord).get().data
+  except KeyError as exc:
+    @[]
+
+proc request*(rdv: RendezVous,
+              ns: string,
+              l: int = DiscoverLimit.int): Future[seq[PeerRecord]] {.async.} =
+  let nsSalted = ns & rdv.salt
+  var
+    s: Table[PeerId, (PeerRecord, Register)]
+    limit: uint64
+    d = Discover(ns: ns)
+
+  if l <= 0 or l > DiscoverLimit.int:
+    raise newException(RendezVousError, "Invalid limit")
+  if ns.len notin 0..255:
+    raise newException(RendezVousError, "Invalid namespace")
+  limit = l.uint64
+  proc requestPeer(peer: PeerId) {.async.} =
+    let conn = await rdv.switch.dial(peer, RendezVousCodec)
+    defer: await conn.close()
+    d.limit = some(limit)
+    d.cookie =
+      try:
+        some(rdv.cookiesSaved[peer][ns])
+      except KeyError as exc:
+        none(seq[byte])
+    await conn.writeLp(encode(Message(
+      msgType: MessageType.Discover,
+      discover: some(d))).buffer)
+    let
+      buf = await conn.readLp(65536)
+      msgRcv = Message.decode(buf).get()
+    if msgRcv.msgType != MessageType.DiscoverResponse or
+       msgRcv.discoverResponse.isNone():
+      debug "Unexpected discover response", msgType = msgRcv.msgType
+      return
+    let resp = msgRcv.discoverResponse.get()
+    if resp.status != ResponseStatus.Ok:
+      trace "Cannot discover", ns, status = resp.status, text = resp.text
+      return
+    if resp.cookie.isSome() and resp.cookie.get().len < 1000:
+      if rdv.cookiesSaved.hasKeyOrPut(peer, {ns: resp.cookie.get()}.toTable):
+        rdv.cookiesSaved[peer][ns] = resp.cookie.get()
+    for r in resp.registrations:
+      if limit == 0: return
+      if r.ttl.isNone() or r.ttl.get() > MaximumTTL: continue
+      let sprRes = SignedPeerRecord.decode(r.signedPeerRecord)
+      if sprRes.isErr(): continue
+      let pr = sprRes.get().data
+      if s.hasKey(pr.peerId):
+        let (prSaved, rSaved) = s[pr.peerId]
+        if (prSaved.seqNo == pr.seqNo and rSaved.ttl.get() < r.ttl.get()) or
+          prSaved.seqNo < pr.seqNo:
+          s[pr.peerId] = (pr, r)
+      else:
+        s[pr.peerId] = (pr, r)
+      limit.dec()
+    for (_, r) in s.values():
+      rdv.save(ns, peer, r, false)
+
+  for peer in rdv.peers:
+    if limit == 0: break
+    if RendezVousCodec notin rdv.switch.peerStore[ProtoBook][peer]: continue
+    try:
+      trace "Send Request", peerId = peer, ns
+      await peer.requestPeer()
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception catch in request", error = exc.msg
+  return toSeq(s.values()).mapIt(it[0])
+
+proc unsubscribeLocally*(rdv: RendezVous, ns: string) =
+  let nsSalted = ns & rdv.salt
+  try:
+    for index in rdv.namespaces[nsSalted]:
+      if rdv.registered[index].peerId == rdv.switch.peerInfo.peerId:
+        rdv.registered[index].expiration = rdv.defaultDT
+  except KeyError:
+    return
+
+proc unsubscribe*(rdv: RendezVous, ns: string) {.async.} =
+  # TODO: find a way to improve this, maybe something similar to the advertise
+  if ns.len notin 1..255:
+    raise newException(RendezVousError, "Invalid namespace")
+  rdv.unsubscribeLocally(ns)
+  let msg = encode(Message(
+    msgType: MessageType.Unregister,
+    unregister: some(Unregister(ns: ns))))
+
+  proc unsubscribePeer(rdv: RendezVous, peerId: PeerId) {.async.} =
+    try:
+      let conn = await rdv.switch.dial(peerId, RendezVousCodec)
+      defer: await conn.close()
+      await conn.writeLp(msg.buffer)
+    except CatchableError as exc:
+      trace "exception while unsubscribing", error = exc.msg
+
+  for peer in rdv.peers:
+    discard await rdv.unsubscribePeer(peer).withTimeout(5.seconds)
+
+proc setup*(rdv: RendezVous, switch: Switch) =
+  rdv.switch = switch
+  proc handlePeer(peerId: PeerId, event: PeerEvent) {.async.} =
+    if event.kind == PeerEventKind.Joined:
+      rdv.peers.add(peerId)
+    elif event.kind == PeerEventKind.Left:
+      rdv.peers.keepItIf(it != peerId)
+  rdv.switch.addPeerEventHandler(handlePeer, Joined)
+  rdv.switch.addPeerEventHandler(handlePeer, Left)
+
+proc new*(T: typedesc[RendezVous],
+          rng: ref HmacDrbgContext = newRng()): T =
+  let rdv = T(
+    rng: rng,
+    salt: string.fromBytes(generateBytes(rng[], 8)),
+    registered: initOffsettedSeq[RegisteredData](1),
+    defaultDT: Moment.now() - 1.days,
+    #registerEvent: newAsyncEvent(),
+    sema: newAsyncSemaphore(SemaphoreDefaultSize)
+  )
+  logScope: topics = "libp2p discovery rendezvous"
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      let
+        buf = await conn.readLp(4096)
+        msg = Message.decode(buf).get()
+      case msg.msgType:
+        of MessageType.Register: await rdv.register(conn, msg.register.get())
+        of MessageType.RegisterResponse:
+          trace "Got an unexpected Register Response", response = msg.registerResponse
+        of MessageType.Unregister: rdv.unregister(conn, msg.unregister.get())
+        of MessageType.Discover: await rdv.discover(conn, msg.discover.get())
+        of MessageType.DiscoverResponse:
+          trace "Got an unexpected Discover Response", response = msg.discoverResponse
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in rendezvous handler", error = exc.msg
+    finally:
+      await conn.close()
+
+  rdv.handler = handleStream
+  rdv.codec = RendezVousCodec
+  return rdv
+
+proc new*(T: typedesc[RendezVous],
+         switch: Switch,
+         rng: ref HmacDrbgContext = newRng()): T =
+  let rdv = T.new(rng)
+  rdv.setup(switch)
+  return rdv
+
+proc deletesRegister(rdv: RendezVous) {.async.} =
+  heartbeat "Register timeout", 1.minutes:
+    let n = Moment.now()
+    rdv.registered.flushIfIt(it.expiration < n)
+    for data in rdv.namespaces.mvalues():
+      data.keepItIf(it >= rdv.registered.offset)
+
+method start*(rdv: RendezVous) {.async.} =
+  if not rdv.registerDeletionLoop.isNil:
+    warn "Starting rendezvous twice"
+    return
+  rdv.registerDeletionLoop = rdv.deletesRegister()
+  rdv.started = true
+
+method stop*(rdv: RendezVous) {.async.} =
+  if rdv.registerDeletionLoop.isNil:
+    warn "Stopping rendezvous without starting it"
+    return
+  rdv.started = false
+  rdv.registerDeletionLoop.cancel()
+  rdv.registerDeletionLoop = nil

libp2p/protocols/secure/noise.nim

@@ -38,7 +38,7 @@ const
   # https://godoc.org/github.com/libp2p/go-libp2p-noise#pkg-constants
   NoiseCodec* = "/noise"
 
-  PayloadString = "noise-libp2p-static-key:"
+  PayloadString = toBytes("noise-libp2p-static-key:")
 
   ProtocolXXName = "Noise_XX_25519_ChaChaPoly_SHA256"
 
@@ -339,7 +339,6 @@ proc handshakeXXOutbound(
     hs = HandshakeState.init()
 
   try:
-
     hs.ss.mixHash(p.commonPrologue)
     hs.s = p.noiseKeys
 
@@ -445,7 +444,6 @@ method readMessage*(sconn: NoiseConnection): Future[seq[byte]] {.async.} =
       dumpMessage(sconn, FlowDirection.Incoming, [])
     trace "Received 0-length message", sconn
 
-
 proc encryptFrame(
     sconn: NoiseConnection,
     cipherFrame: var openArray[byte],
@@ -506,7 +504,7 @@ method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] =
   # sequencing issues
   sconn.stream.write(cipherFrames)
 
-method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureConn] {.async.} =
+method handshake*(p: Noise, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} =
   trace "Starting Noise handshake", conn, initiator
 
   let timeout = conn.timeout
@@ -515,7 +513,7 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
   # https://github.com/libp2p/specs/tree/master/noise#libp2p-data-in-handshake-messages
   let
     signedPayload = p.localPrivateKey.sign(
-      PayloadString.toBytes & p.noiseKeys.publicKey.getBytes).tryGet()
+      PayloadString & p.noiseKeys.publicKey.getBytes).tryGet()
 
   var
     libp2pProof = initProtoBuffer()
@@ -538,11 +536,9 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
       remoteSig: Signature
       remoteSigBytes: seq[byte]
 
-    let r1 = remoteProof.getField(1, remotePubKeyBytes)
+    if not remoteProof.getField(1, remotePubKeyBytes).valueOr(false):
-    let r2 = remoteProof.getField(2, remoteSigBytes)
-    if r1.isErr() or not(r1.get()):
       raise newException(NoiseHandshakeError, "Failed to deserialize remote public key bytes. (initiator: " & $initiator & ")")
-    if r2.isErr() or not(r2.get()):
+    if not remoteProof.getField(2, remoteSigBytes).valueOr(false):
       raise newException(NoiseHandshakeError, "Failed to deserialize remote signature bytes. (initiator: " & $initiator & ")")
 
     if not remotePubKey.init(remotePubKeyBytes):
@@ -550,33 +546,34 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
     if not remoteSig.init(remoteSigBytes):
       raise newException(NoiseHandshakeError, "Failed to decode remote signature. (initiator: " & $initiator & ")")
 
-    let verifyPayload = PayloadString.toBytes & handshakeRes.rs.getBytes
+    let verifyPayload = PayloadString & handshakeRes.rs.getBytes
     if not remoteSig.verify(verifyPayload, remotePubKey):
       raise newException(NoiseHandshakeError, "Noise handshake signature verify failed.")
     else:
       trace "Remote signature verified", conn
 
-    if initiator:
+    let pid = PeerId.init(remotePubKey).valueOr:
-      let pid = PeerId.init(remotePubKey)
+      raise newException(NoiseHandshakeError, "Invalid remote peer id: " & $error)
-      if not conn.peerId.validate():
+
-        raise newException(NoiseHandshakeError, "Failed to validate peerId.")
+    trace "Remote peer id", pid = $pid
-      if pid.isErr or pid.get() != conn.peerId:
+
+    if peerId.isSome():
+      let targetPid = peerId.get()
+      if not targetPid.validate():
+        raise newException(NoiseHandshakeError, "Failed to validate expected peerId.")
+
+      if pid != targetPid:
         var
           failedKey: PublicKey
-        discard extractPublicKey(conn.peerId, failedKey)
+        discard extractPublicKey(targetPid, failedKey)
-        debug "Noise handshake, peer infos don't match!",
+        debug "Noise handshake, peer id doesn't match!",
           initiator, dealt_peer = conn,
           dealt_key = $failedKey, received_peer = $pid,
           received_key = $remotePubKey
-        raise newException(NoiseHandshakeError, "Noise handshake, peer infos don't match! " & $pid & " != " & $conn.peerId)
+        raise newException(NoiseHandshakeError, "Noise handshake, peer id don't match! " & $pid & " != " & $targetPid)
-    else:
+    conn.peerId = pid
-      let pid = PeerId.init(remotePubKey)
-      if pid.isErr:
-        raise newException(NoiseHandshakeError, "Invalid remote peer id")
-      conn.peerId = pid.get()
 
     var tmp = NoiseConnection.new(conn, conn.peerId, conn.observedAddr)
-
     if initiator:
       tmp.readCs = handshakeRes.cs2
       tmp.writeCs = handshakeRes.cs1

libp2p/protocols/secure/secio.nim

@@ -291,7 +291,7 @@ proc transactMessage(conn: Connection,
   await conn.write(msg)
   return await conn.readRawMessage()
 
-method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[SecureConn] {.async.} =
+method handshake*(s: Secio, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} =
   var
     localNonce: array[SecioNonceSize, byte]
     remoteNonce: seq[byte]
@@ -342,8 +342,13 @@ method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[S
 
   remotePeerId = PeerId.init(remotePubkey).tryGet()
 
-  # TODO: PeerId check against supplied PeerId
+  if peerId.isSome():
-  if not initiator:
+    let targetPid = peerId.get()
+    if not targetPid.validate():
+      raise newException(SecioError, "Failed to validate expected peerId.")
+
+    if remotePeerId != targetPid:
+      raise newException(SecioError, "Peer ids don't match!")
   conn.peerId = remotePeerId
   let order = getOrder(remoteBytesPubkey, localNonce, localBytesPubkey,
                        remoteNonce).tryGet()

libp2p/protocols/secure/secure.nim

@@ -13,6 +13,7 @@ else:
   {.push raises: [].}
 
 import std/[strformat]
+import stew/results
 import chronos, chronicles
 import ../protocol,
        ../../stream/streamseq,
@@ -21,7 +22,7 @@ import ../protocol,
        ../../peerinfo,
        ../../errors
 
-export protocol
+export protocol, results
 
 logScope:
   topics = "libp2p secure"
@@ -48,7 +49,7 @@ chronicles.formatIt(SecureConn): shortLog(it)
 proc new*(T: type SecureConn,
            conn: Connection,
            peerId: PeerId,
-           observedAddr: MultiAddress,
+           observedAddr: Opt[MultiAddress],
            timeout: Duration = DefaultConnectionTimeout): T =
   result = T(stream: conn,
              peerId: peerId,
@@ -79,13 +80,15 @@ method getWrapped*(s: SecureConn): Connection = s.stream
 
 method handshake*(s: Secure,
                   conn: Connection,
-                  initiator: bool): Future[SecureConn] {.async, base.} =
+                  initiator: bool,
+                  peerId: Opt[PeerId]): Future[SecureConn] {.async, base.} =
   doAssert(false, "Not implemented!")
 
 proc handleConn(s: Secure,
                  conn: Connection,
-                 initiator: bool): Future[Connection] {.async.} =
+                 initiator: bool,
-  var sconn = await s.handshake(conn, initiator)
+                 peerId: Opt[PeerId]): Future[Connection] {.async.} =
+  var sconn = await s.handshake(conn, initiator, peerId)
   # mark connection bottom level transport direction
   # this is the safest place to do this
   # we require this information in for example gossipsub
@@ -121,7 +124,7 @@ method init*(s: Secure) =
     try:
       # We don't need the result but we
       # definitely need to await the handshake
-      discard await s.handleConn(conn, false)
+      discard await s.handleConn(conn, false, Opt.none(PeerId))
       trace "connection secured", conn
     except CancelledError as exc:
       warn "securing connection canceled", conn
@@ -135,9 +138,10 @@ method init*(s: Secure) =
 
 method secure*(s: Secure,
                conn: Connection,
-               initiator: bool):
+               initiator: bool,
+               peerId: Opt[PeerId]):
                Future[Connection] {.base.} =
-  s.handleConn(conn, initiator)
+  s.handleConn(conn, initiator, peerId)
 
 method readOnce*(s: SecureConn,
                  pbytes: pointer,

libp2p/stream/bufferstream.nim

@@ -79,7 +79,7 @@ method pushData*(s: BufferStream, data: seq[byte]) {.base, async.} =
     &"Only one concurrent push allowed for stream {s.shortLog()}")
 
   if s.isClosed or s.pushedEof:
-    raise newLPStreamEOFError()
+    raise newLPStreamClosedError()
 
   if data.len == 0:
     return # Don't push 0-length buffers, these signal EOF

libp2p/stream/chronosstream.nim

@@ -13,10 +13,13 @@ else:
   {.push raises: [].}
 
 import std/[oids, strformat]
+import stew/results
 import chronos, chronicles, metrics
 import connection
 import ../utility
 
+export results
+
 logScope:
   topics = "libp2p chronosstream"
 
@@ -60,7 +63,7 @@ proc init*(C: type ChronosStream,
            client: StreamTransport,
            dir: Direction,
            timeout = DefaultChronosStreamTimeout,
-           observedAddr: MultiAddress = MultiAddress()): ChronosStream =
+           observedAddr: Opt[MultiAddress]): ChronosStream =
   result = C(client: client,
              timeout: timeout,
              dir: dir,
@@ -127,6 +130,9 @@ proc completeWrite(
 method write*(s: ChronosStream, msg: seq[byte]): Future[void] =
   # Avoid a copy of msg being kept in the closure created by `{.async.}` as this
   # drives up memory usage
+  if msg.len == 0:
+    trace "Empty byte seq, nothing to write"
+    return
   if s.closed:
     let fut = newFuture[void]("chronosstream.write.closed")
     fut.fail(newLPStreamClosedError())

libp2p/stream/connection.nim

@@ -13,13 +13,14 @@ else:
   {.push raises: [].}
 
 import std/[hashes, oids, strformat]
+import stew/results
 import chronicles, chronos, metrics
 import lpstream,
        ../multiaddress,
        ../peerinfo,
        ../errors
 
-export lpstream, peerinfo, errors
+export lpstream, peerinfo, errors, results
 
 logScope:
   topics = "libp2p connection"
@@ -37,7 +38,7 @@ type
     timerTaskFut: Future[void]      # the current timer instance
     timeoutHandler*: TimeoutHandler # timeout handler
     peerId*: PeerId
-    observedAddr*: MultiAddress
+    observedAddr*: Opt[MultiAddress]
     upgraded*: Future[void]
     protocol*: string               # protocol used by the connection, used as tag for metrics
     transportDir*: Direction        # The bottom level transport (generally the socket) direction
@@ -160,9 +161,9 @@ method getWrapped*(s: Connection): Connection {.base.} =
 proc new*(C: type Connection,
            peerId: PeerId,
            dir: Direction,
+           observedAddr: Opt[MultiAddress],
            timeout: Duration = DefaultConnectionTimeout,
-           timeoutHandler: TimeoutHandler = nil,
+           timeoutHandler: TimeoutHandler = nil): Connection =
-           observedAddr: MultiAddress = MultiAddress()): Connection =
   result = C(peerId: peerId,
              dir: dir,
              timeout: timeout,

libp2p/stream/lpstream.nim

@@ -59,7 +59,18 @@ type
   LPStreamWriteError* = object of LPStreamError
     par*: ref CatchableError
   LPStreamEOFError* = object of LPStreamError
-  LPStreamClosedError* = object of LPStreamError
+
+#        X        |           Read            |         Write
+#   Local close   |           Works           |  LPStreamClosedError
+#   Remote close  | LPStreamRemoteClosedError |         Works
+#   Local reset   |    LPStreamClosedError    |  LPStreamClosedError
+#   Remote reset  |    LPStreamResetError     |  LPStreamResetError
+# Connection down |     LPStreamConnDown      | LPStreamConnDownError
+
+  LPStreamResetError* = object of LPStreamEOFError
+  LPStreamClosedError* = object of LPStreamEOFError
+  LPStreamRemoteClosedError* = object of LPStreamEOFError
+  LPStreamConnDownError* = object of LPStreamEOFError
 
   InvalidVarintError* = object of LPStreamError
   MaxSizeError* = object of LPStreamError
@@ -119,9 +130,22 @@ proc newLPStreamIncorrectDefect*(m: string): ref LPStreamIncorrectDefect =
 proc newLPStreamEOFError*(): ref LPStreamEOFError =
   result = newException(LPStreamEOFError, "Stream EOF!")
 
+proc newLPStreamResetError*(): ref LPStreamResetError =
+  result = newException(LPStreamResetError, "Stream Reset!")
+
 proc newLPStreamClosedError*(): ref LPStreamClosedError =
   result = newException(LPStreamClosedError, "Stream Closed!")
 
+proc newLPStreamRemoteClosedError*(): ref LPStreamRemoteClosedError =
+  result = newException(LPStreamRemoteClosedError, "Stream Remotely Closed!")
+
+proc newLPStreamConnDownError*(
+    parentException: ref Exception = nil): ref LPStreamConnDownError =
+  result = newException(
+    LPStreamConnDownError,
+    "Stream Underlying Connection Closed!",
+    parentException)
+
 func shortLog*(s: LPStream): auto =
   if s.isNil: "LPStream(nil)"
   else: $s.oid
@@ -165,6 +189,8 @@ proc readExactly*(s: LPStream,
   ## Waits for `nbytes` to be available, then read
   ## them and return them
   if s.atEof:
+    var ch: char
+    discard await s.readOnce(addr ch, 1)
     raise newLPStreamEOFError()
 
   if nbytes == 0:
@@ -183,6 +209,10 @@ proc readExactly*(s: LPStream,
   if read == 0:
     doAssert s.atEof()
     trace "couldn't read all bytes, stream EOF", s, nbytes, read
+    # Re-readOnce to raise a more specific error than EOF
+    # Raise EOF if it doesn't raise anything(shouldn't happen)
+    discard await s.readOnce(addr pbuffer[read], nbytes - read)
+    warn "Read twice while at EOF"
     raise newLPStreamEOFError()
 
   if read < nbytes:
@@ -200,8 +230,7 @@ proc readLine*(s: LPStream,
 
   while true:
     var ch: char
-    if (await readOnce(s, addr ch, 1)) == 0:
+    await readExactly(s, addr ch, 1)
-      raise newLPStreamEOFError()
 
     if sep[state] == ch:
       inc(state)
@@ -224,8 +253,7 @@ proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe, public.} =
     buffer: array[10, byte]
 
   for i in 0..<len(buffer):
-    if (await conn.readOnce(addr buffer[i], 1)) == 0:
+    await conn.readExactly(addr buffer[i], 1)
-      raise newLPStreamEOFError()
 
     var
       varint: uint64

libp2p/switch.nim

@@ -128,6 +128,13 @@ method connect*(
 
   s.dialer.connect(peerId, addrs, forceDial)
 
+method connect*(
+  s: Switch,
+  addrs: seq[MultiAddress]): Future[PeerId] =
+  ## Connects to a peer and retrieve its PeerId
+
+  s.dialer.connect(addrs)
+
 method dial*(
   s: Switch,
   peerId: PeerId,
@@ -292,11 +299,11 @@ proc start*(s: Switch) {.async, gcsafe, public.} =
   trace "starting switch for peer", peerInfo = s.peerInfo
   var startFuts: seq[Future[void]]
   for t in s.transports:
-    let addrs = s.peerInfo.addrs.filterIt(
+    let addrs = s.peerInfo.listenAddrs.filterIt(
       t.handles(it)
     )
 
-    s.peerInfo.addrs.keepItIf(
+    s.peerInfo.listenAddrs.keepItIf(
       it notin addrs
     )
 
@@ -305,19 +312,17 @@ proc start*(s: Switch) {.async, gcsafe, public.} =
 
   await allFutures(startFuts)
 
-  for s in startFuts:
+  for fut in startFuts:
-    if s.failed:
+    if fut.failed:
-      # TODO: replace this exception with a `listenError` callback. See
+      await s.stop()
-      # https://github.com/status-im/nim-libp2p/pull/662 for more info.
+      raise fut.error
-      raise newException(transport.TransportError,
-        "Failed to start one transport", s.error)
 
   for t in s.transports: # for each transport
     if t.addrs.len > 0 or t.running:
       s.acceptFuts.add(s.accept(t))
-      s.peerInfo.addrs &= t.addrs
+      s.peerInfo.listenAddrs &= t.addrs
 
-  s.peerInfo.update()
+  await s.peerInfo.update()
 
   await s.ms.start()
 

libp2p/transports/tcptransport.nim

@@ -15,6 +15,7 @@ else:
   {.push raises: [].}
 
 import std/[oids, sequtils]
+import stew/results
 import chronos, chronicles
 import transport,
        ../errors,
@@ -31,7 +32,7 @@ import transport,
 logScope:
   topics = "libp2p tcptransport"
 
-export transport
+export transport, results
 
 const
   TcpTransportTrackerName* = "libp2p.tcptransport"
@@ -71,18 +72,20 @@ proc setupTcpTransportTracker(): TcpTransportTracker =
   result.isLeaked = leakTransport
   addTracker(TcpTransportTrackerName, result)
 
-proc connHandler*(self: TcpTransport,
+proc getObservedAddr(client: StreamTransport): Future[MultiAddress] {.async.} =
-                  client: StreamTransport,
-                  dir: Direction): Future[Connection] {.async.} =
-  var observedAddr: MultiAddress = MultiAddress()
   try:
-    observedAddr = MultiAddress.init(client.remoteAddress).tryGet()
+    return MultiAddress.init(client.remoteAddress).tryGet()
   except CatchableError as exc:
     trace "Failed to create observedAddr", exc = exc.msg
     if not(isNil(client) and client.closed):
       await client.closeWait()
     raise exc
 
+proc connHandler*(self: TcpTransport,
+                  client: StreamTransport,
+                  observedAddr: Opt[MultiAddress],
+                  dir: Direction): Future[Connection] {.async.} =
+
   trace "Handling tcp connection", address = $observedAddr,
                                    dir = $dir,
                                    clients = self.clients[Direction.In].len +
@@ -222,7 +225,8 @@ method accept*(self: TcpTransport): Future[Connection] {.async, gcsafe.} =
     self.acceptFuts[index] = self.servers[index].accept()
 
     let transp = await finished
-    return await self.connHandler(transp, Direction.In)
+    let observedAddr = await getObservedAddr(transp)
+    return await self.connHandler(transp, Opt.some(observedAddr), Direction.In)
   except TransportOsError as exc:
     # TODO: it doesn't sound like all OS errors
     # can  be ignored, we should re-raise those
@@ -250,7 +254,8 @@ method dial*(
 
   let transp = await connect(address)
   try:
-    return await self.connHandler(transp, Direction.Out)
+    let observedAddr = await getObservedAddr(transp)
+    return await self.connHandler(transp, Opt.some(observedAddr), Direction.Out)
   except CatchableError as err:
     await transp.closeWait()
     raise err

libp2p/transports/transport.nim

@@ -87,12 +87,13 @@ method upgradeIncoming*(
 
 method upgradeOutgoing*(
   self: Transport,
-  conn: Connection): Future[Connection] {.base, gcsafe.} =
+  conn: Connection,
+  peerId: Opt[PeerId]): Future[Connection] {.base, gcsafe.} =
   ## base upgrade method that the transport uses to perform
   ## transport specific upgrades
   ##
 
-  self.upgrader.upgradeOutgoing(conn)
+  self.upgrader.upgradeOutgoing(conn, peerId)
 
 method handles*(
   self: Transport,

libp2p/transports/wstransport.nim

@@ -15,6 +15,7 @@ else:
   {.push raises: [].}
 
 import std/[sequtils]
+import stew/results
 import chronos, chronicles
 import transport,
        ../errors,
@@ -31,7 +32,7 @@ import transport,
 logScope:
   topics = "libp2p wstransport"
 
-export transport, websock
+export transport, websock, results
 
 const
   WsTransportTrackerName* = "libp2p.wstransport"
@@ -45,8 +46,8 @@ type
 proc new*(T: type WsStream,
            session: WSSession,
            dir: Direction,
-           timeout = 10.minutes,
+           observedAddr: Opt[MultiAddress],
-           observedAddr: MultiAddress = MultiAddress()): T =
+           timeout = 10.minutes): T =
 
   let stream = T(
     session: session,
@@ -221,8 +222,7 @@ proc connHandler(self: WsTransport,
         await stream.close()
       raise exc
 
-  let conn = WsStream.new(stream, dir)
+  let conn = WsStream.new(stream, dir, Opt.some(observedAddr))
-  conn.observedAddr = observedAddr
 
   self.connections[dir].add(conn)
   proc onClose() {.async.} =

libp2p/upgrademngrs/muxedupgrade.nim

@@ -88,10 +88,11 @@ proc mux*(
 
 method upgradeOutgoing*(
   self: MuxedUpgrade,
-  conn: Connection): Future[Connection] {.async, gcsafe.} =
+  conn: Connection,
+  peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} =
   trace "Upgrading outgoing connection", conn
 
-  let sconn = await self.secure(conn) # secure the connection
+  let sconn = await self.secure(conn, peerId) # secure the connection
   if isNil(sconn):
     raise newException(UpgradeFailedError,
       "unable to secure connection, stopping upgrade")
@@ -129,7 +130,7 @@ method upgradeIncoming*(
 
     var cconn = conn
     try:
-      var sconn = await secure.secure(cconn, false)
+      var sconn = await secure.secure(cconn, false, Opt.none(PeerId))
       if isNil(sconn):
         return
 

libp2p/upgrademngrs/upgrade.nim

@@ -47,12 +47,14 @@ method upgradeIncoming*(
 
 method upgradeOutgoing*(
   self: Upgrade,
-  conn: Connection): Future[Connection] {.base.} =
+  conn: Connection,
+  peerId: Opt[PeerId]): Future[Connection] {.base.} =
   doAssert(false, "Not implemented!")
 
 proc secure*(
   self: Upgrade,
-  conn: Connection): Future[Connection] {.async, gcsafe.} =
+  conn: Connection,
+  peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} =
   if self.secureManagers.len <= 0:
     raise newException(UpgradeFailedError, "No secure managers registered!")
 
@@ -67,7 +69,7 @@ proc secure*(
   # let's avoid duplicating checks but detect if it fails to do it properly
   doAssert(secureProtocol.len > 0)
 
-  return await secureProtocol[0].secure(conn, true)
+  return await secureProtocol[0].secure(conn, true, peerId)
 
 proc identify*(
   self: Upgrade,

libp2p/utils/heartbeat.nim

@@ -25,6 +25,14 @@ template heartbeat*(name: string, interval: Duration, body: untyped): untyped =
     nextHeartbeat += interval
     let now = Moment.now()
     if nextHeartbeat < now:
-      info "Missed heartbeat", heartbeat = name, delay = now - nextHeartbeat
+      let
-      nextHeartbeat = now + interval
+        delay = now - nextHeartbeat
+        itv = interval
+      if delay > itv:
+        info "Missed multiple heartbeats", heartbeat = name,
+          delay = delay, hinterval = itv
+      else:
+        debug "Missed heartbeat", heartbeat = name,
+          delay = delay, hinterval = itv
+      nextHeartbeat = now + itv
     await sleepAsync(nextHeartbeat - now)

libp2p/utils/offsettedseq.nim

@@ -0,0 +1,73 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import sequtils
+
+type
+  OffsettedSeq*[T] = object
+    s*: seq[T]
+    offset*: int
+
+proc initOffsettedSeq*[T](offset: int = 0): OffsettedSeq[T] =
+  OffsettedSeq[T](s: newSeq[T](), offset: offset)
+
+proc all*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool =
+  o.s.all(pred)
+
+proc any*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool =
+  o.s.any(pred)
+
+proc apply*[T](o: OffsettedSeq[T], op: proc (x: T)) =
+  o.s.apply(pred)
+
+proc apply*[T](o: OffsettedSeq[T], op: proc (x: T): T) =
+  o.s.apply(pred)
+
+proc apply*[T](o: OffsettedSeq[T], op: proc (x: var T)) =
+  o.s.apply(pred)
+
+func count*[T](o: OffsettedSeq[T], x: T): int =
+  o.s.count(x)
+
+proc flushIf*[T](o: OffsettedSeq[T], pred: proc (x: T): bool) =
+  var i = 0
+  for e in o.s:
+    if not pred(e): break
+    i.inc()
+  if i > 0:
+    o.s.delete(0..<i)
+    o.offset.inc(i)
+
+template flushIfIt*(o, pred: untyped) =
+  var i = 0
+  for it {.inject.} in o.s:
+    if not pred: break
+    i.inc()
+  if i > 0:
+    when (NimMajor, NimMinor) < (1, 4):
+      o.s.delete(0, i - 1)
+    else:
+      o.s.delete(0..<i)
+    o.offset.inc(i)
+
+proc add*[T](o: var OffsettedSeq[T], v: T) =
+  o.s.add(v)
+
+proc `[]`*[T](o: var OffsettedSeq[T], index: int): var T =
+  o.s[index - o.offset]
+
+iterator items*[T](o: OffsettedSeq[T]): T =
+  for e in o.s:
+    yield e
+
+proc high*[T](o: OffsettedSeq[T]): int =
+  o.s.high + o.offset
+
+proc low*[T](o: OffsettedSeq[T]): int =
+  o.s.low + o.offset

libp2p/vbuffer.nim

@@ -39,21 +39,9 @@ proc len*(vb: VBuffer): int =
   result = len(vb.buffer) - vb.offset
   doAssert(result >= 0)
 
-proc isLiteral[T](s: seq[T]): bool {.inline.} =
-  when defined(gcOrc) or defined(gcArc):
-    false
-  else:
-    type
-      SeqHeader = object
-        length, reserved: int
-    (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0
-
 proc initVBuffer*(data: seq[byte], offset = 0): VBuffer =
   ## Initialize VBuffer with shallow copy of ``data``.
-  if isLiteral(data):
   result.buffer = data
-  else:
-    shallowCopy(result.buffer, data)
   result.offset = offset
 
 proc initVBuffer*(data: openArray[byte], offset = 0): VBuffer =

mkdocs.yml

@@ -3,7 +3,9 @@ site_name: nim-libp2p
 repo_url: https://github.com/status-im/nim-libp2p
 repo_name: status-im/nim-libp2p
 site_url: https://status-im.github.io/nim-libp2p/docs
-edit_uri: edit/unstable/examples/
+# Can't find a way to point the edit to the .nim instead
+# of the .md
+edit_uri: ''
 
 docs_dir: examples
 
@@ -40,6 +42,9 @@ theme:
 nav:
   - Introduction: README.md
   - Tutorials:
-    - 'Part I: Simple connection': tutorial_1_connect.md
+    - 'Simple connection': tutorial_1_connect.md
-    - 'Part II: Custom protocol': tutorial_2_customproto.md
+    - 'Create a custom protocol': tutorial_2_customproto.md
+    - 'Protobuf': tutorial_3_protobuf.md
+    - 'GossipSub': tutorial_4_gossipsub.md
+    - 'Circuit Relay': circuitrelay.md
   - Reference: '/nim-libp2p/master/libp2p.html'

tests/commoninterop.nim

@@ -3,7 +3,7 @@ import chronos, chronicles, stew/byteutils
 import helpers
 import ../libp2p
 import ../libp2p/[daemon/daemonapi, varint, transports/wstransport, crypto/crypto]
-import ../libp2p/protocols/relay/[relay, client, utils]
+import ../libp2p/protocols/connectivity/relay/[relay, client, utils]
 
 type
   SwitchCreator = proc(

tests/commontransport.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import sequtils
-import chronos, stew/byteutils
+import chronos, stew/[byteutils, results]
 import ../libp2p/[stream/connection,
                   transports/transport,
                   upgrademngrs/upgrade,
@@ -35,14 +35,16 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
 
       proc acceptHandler() {.async, gcsafe.} =
         let conn = await transport1.accept()
-        check transport1.handles(conn.observedAddr)
+        if conn.observedAddr.isSome():
+          check transport1.handles(conn.observedAddr.get())
         await conn.close()
 
       let handlerWait = acceptHandler()
 
       let conn = await transport2.dial(transport1.addrs[0])
 
-      check transport2.handles(conn.observedAddr)
+      if conn.observedAddr.isSome():
+        check transport2.handles(conn.observedAddr.get())
 
       await conn.close() #for some protocols, closing requires actively reading, so we must close here
 

tests/pubsub/testgossipinternal.nim

@@ -22,10 +22,7 @@ proc getPubSubPeer(p: TestGossipSub, peerId: PeerId): PubSubPeer =
   proc getConn(): Future[Connection] =
     p.switch.dial(peerId, GossipSubCodec)
 
-  proc dropConn(peer: PubSubPeer) =
+  let pubSubPeer = PubSubPeer.new(peerId, getConn, nil, GossipSubCodec, 1024 * 1024)
-    discard # we don't care about it here yet
-
-  let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, nil, GossipSubCodec, 1024 * 1024)
   debug "created new pubsub peer", peerId
 
   p.peers[peerId] = pubSubPeer

tests/pubsub/testpubsub.nim

@@ -2,9 +2,11 @@
 
 import ../stublogger
 
-import testfloodsub,
+import testfloodsub
-       testgossipsub,
+when not defined(linux):
-       testgossipsub2,
+  import testgossipsub,
+         testgossipsub2
+import
        testmcache,
        testtimedcache,
        testmessage

tests/testautonat.nim

@@ -3,7 +3,7 @@ import chronos
 import
   ../libp2p/[
     builders,
-    protocols/autonat
+    protocols/connectivity/autonat
   ],
   ./helpers
 

tests/testconnmngr.nim

@@ -1,4 +1,5 @@
 import sequtils
+import stew/results
 import chronos
 import ../libp2p/[connmanager,
                   stream/connection,
@@ -9,6 +10,9 @@ import ../libp2p/[connmanager,
 
 import helpers
 
+proc getConnection(peerId: PeerId, dir: Direction = Direction.In): Connection = 
+  return Connection.new(peerId, dir, Opt.none(MultiAddress))
+
 type
   TestMuxer = ref object of Muxer
     peerId: PeerId
@@ -18,7 +22,7 @@ method newStream*(
   name: string = "",
   lazy: bool = false):
   Future[Connection] {.async, gcsafe.} =
-  result = Connection.new(m.peerId, Direction.Out)
+  result = getConnection(m.peerId, Direction.Out)
 
 suite "Connection Manager":
   teardown:
@@ -27,7 +31,7 @@ suite "Connection Manager":
   asyncTest "add and retrieve a connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
 
     connMngr.storeConn(conn)
     check conn in connMngr
@@ -41,7 +45,7 @@ suite "Connection Manager":
   asyncTest "shouldn't allow a closed connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
     await conn.close()
 
     expect CatchableError:
@@ -52,7 +56,7 @@ suite "Connection Manager":
   asyncTest "shouldn't allow an EOFed connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
     conn.isEof = true
 
     expect CatchableError:
@@ -64,7 +68,7 @@ suite "Connection Manager":
   asyncTest "add and retrieve a muxer":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
     let muxer = new Muxer
     muxer.connection = conn
 
@@ -80,7 +84,7 @@ suite "Connection Manager":
   asyncTest "shouldn't allow a muxer for an untracked connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
     let muxer = new Muxer
     muxer.connection = conn
 
@@ -94,8 +98,8 @@ suite "Connection Manager":
   asyncTest "get conn with direction":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn1 = Connection.new(peerId, Direction.Out)
+    let conn1 = getConnection(peerId, Direction.Out)
-    let conn2 = Connection.new(peerId, Direction.In)
+    let conn2 = getConnection(peerId)
 
     connMngr.storeConn(conn1)
     connMngr.storeConn(conn2)
@@ -114,7 +118,7 @@ suite "Connection Manager":
   asyncTest "get muxed stream for peer":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
 
     let muxer = new TestMuxer
     muxer.peerId = peerId
@@ -134,7 +138,7 @@ suite "Connection Manager":
   asyncTest "get stream from directed connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
 
     let muxer = new TestMuxer
     muxer.peerId = peerId
@@ -155,7 +159,7 @@ suite "Connection Manager":
   asyncTest "get stream from any connection":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
 
     let muxer = new TestMuxer
     muxer.peerId = peerId
@@ -175,11 +179,11 @@ suite "Connection Manager":
     let connMngr = ConnManager.new(maxConnsPerPeer = 1)
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
 
-    connMngr.storeConn(Connection.new(peerId, Direction.In))
+    connMngr.storeConn(getConnection(peerId))
 
     let conns = @[
-        Connection.new(peerId, Direction.In),
+        getConnection(peerId),
-        Connection.new(peerId, Direction.In)]
+        getConnection(peerId)]
 
     expect TooManyConnectionsError:
       connMngr.storeConn(conns[0])
@@ -193,7 +197,7 @@ suite "Connection Manager":
   asyncTest "cleanup on connection close":
     let connMngr = ConnManager.new()
     let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
-    let conn = Connection.new(peerId, Direction.In)
+    let conn = getConnection(peerId)
     let muxer = new Muxer
 
     muxer.connection = conn
@@ -220,7 +224,7 @@ suite "Connection Manager":
         Direction.In else:
         Direction.Out
 
-      let conn = Connection.new(peerId, dir)
+      let conn = getConnection(peerId, dir)
       let muxer = new Muxer
       muxer.connection = conn
 
@@ -353,7 +357,7 @@ suite "Connection Manager":
       let slot = await ((connMngr.getOutgoingSlot()).wait(10.millis))
 
       let conn =
-        Connection.new(
+        getConnection(
           PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet(),
           Direction.In)
 

tests/testdiscovery.nim

@@ -0,0 +1,51 @@
+{.used.}
+
+import options, chronos, sets
+import stew/byteutils
+import ../libp2p/[protocols/rendezvous,
+                  switch,
+                  builders,
+                  discovery/discoverymngr,
+                  discovery/rendezvousinterface,]
+import ./helpers
+
+proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withRendezVous(rdv)
+    .build()
+
+suite "Discovery":
+  teardown:
+    checkTrackers()
+  asyncTest "RendezVous test":
+    let
+      rdvA = RendezVous.new()
+      rdvB = RendezVous.new()
+      clientA = createSwitch(rdvA)
+      clientB = createSwitch(rdvB)
+      remoteNode = createSwitch()
+      dmA = DiscoveryManager()
+      dmB = DiscoveryManager()
+    dmA.add(RendezVousInterface.new(rdvA, ttr = 500.milliseconds))
+    dmB.add(RendezVousInterface.new(rdvB))
+    await allFutures(clientA.start(), clientB.start(), remoteNode.start())
+
+    await clientB.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs)
+    await clientA.connect(remoteNode.peerInfo.peerId, remoteNode.peerInfo.addrs)
+
+    dmB.advertise(RdvNamespace("foo"))
+
+    let
+      query = dmA.request(RdvNamespace("foo"))
+      res = await query.getPeer()
+    check:
+      res{PeerId}.get() == clientB.peerInfo.peerId
+      res[PeerId] == clientB.peerInfo.peerId
+      res.getAll(PeerId) == @[clientB.peerInfo.peerId]
+      toHashSet(res.getAll(MultiAddress)) == toHashSet(clientB.peerInfo.addrs)
+    await allFutures(clientA.stop(), clientB.stop(), remoteNode.stop())

tests/testidentify.nim

@@ -52,6 +52,9 @@ suite "Identify":
       msListen = MultistreamSelect.new()
       msDial = MultistreamSelect.new()
 
+      serverFut = transport1.start(ma)
+      await remotePeerInfo.update()
+
     asyncTeardown:
       await conn.close()
       await acceptFut
@@ -61,7 +64,6 @@ suite "Identify":
 
     asyncTest "default agent version":
       msListen.addHandler(IdentifyCodec, identifyProto1)
-      serverFut = transport1.start(ma)
       proc acceptHandler(): Future[void] {.async, gcsafe.} =
         let c = await transport1.accept()
         await msListen.handle(c)
@@ -84,8 +86,6 @@ suite "Identify":
       remotePeerInfo.agentVersion = customAgentVersion
       msListen.addHandler(IdentifyCodec, identifyProto1)
 
-      serverFut = transport1.start(ma)
-
       proc acceptHandler(): Future[void] {.async, gcsafe.} =
         let c = await transport1.accept()
         await msListen.handle(c)
@@ -105,7 +105,6 @@ suite "Identify":
 
     asyncTest "handle failed identify":
       msListen.addHandler(IdentifyCodec, identifyProto1)
-      asyncSpawn transport1.start(ma)
 
       proc acceptHandler() {.async.} =
         var conn: Connection
@@ -128,7 +127,6 @@ suite "Identify":
     asyncTest "can send signed peer record":
       msListen.addHandler(IdentifyCodec, identifyProto1)
       identifyProto1.sendSignedPeerRecord = true
-      serverFut = transport1.start(ma)
       proc acceptHandler(): Future[void] {.async, gcsafe.} =
         let c = await transport1.accept()
         await msListen.handle(c)
@@ -195,7 +193,8 @@ suite "Identify":
 
     asyncTest "simple push identify":
       switch2.peerInfo.protocols.add("/newprotocol/")
-      switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
+      switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
+      await switch2.peerInfo.update()
 
       check:
         switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs
@@ -216,7 +215,8 @@ suite "Identify":
 
     asyncTest "wrong peer id push identify":
       switch2.peerInfo.protocols.add("/newprotocol/")
-      switch2.peerInfo.addrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
+      switch2.peerInfo.listenAddrs.add(MultiAddress.init("/ip4/127.0.0.1/tcp/5555").tryGet())
+      await switch2.peerInfo.update()
 
       check:
         switch1.peerStore[AddressBook][switch2.peerInfo.peerId] != switch2.peerInfo.addrs

tests/testinterop.nim

@@ -2,7 +2,7 @@ import stublogger
 
 import helpers, commoninterop
 import ../libp2p
-import ../libp2p/crypto/crypto, ../libp2p/protocols/relay/[relay, client]
+import ../libp2p/crypto/crypto, ../libp2p/protocols/connectivity/relay/[relay, client]
 
 proc switchMplexCreator(
     ma: MultiAddress = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet(),

tests/testmplex.nim

@@ -119,7 +119,7 @@ suite "Mplex":
       # should still allow reading until buffer EOF
       await chann.readExactly(addr data[3], 3)
 
-      expect LPStreamEOFError:
+      expect LPStreamRemoteClosedError:
         # this should fail now
         await chann.readExactly(addr data[0], 3)
 
@@ -143,7 +143,7 @@ suite "Mplex":
       let readFut = chann.readExactly(addr data[3], 3)
       await allFutures(closeFut, readFut)
 
-      expect LPStreamEOFError:
+      expect LPStreamRemoteClosedError:
         await chann.readExactly(addr data[0], 6) # this should fail now
 
       await chann.close()
@@ -174,7 +174,7 @@ suite "Mplex":
       var buf: array[1, byte]
       check: (await chann.readOnce(addr buf[0], 1)) == 0 # EOF marker read
 
-      expect LPStreamEOFError:
+      expect LPStreamClosedError:
         await chann.pushData(@[byte(1)])
 
       await chann.close()
@@ -190,7 +190,7 @@ suite "Mplex":
 
       await chann.reset()
       var data = newSeq[byte](1)
-      expect LPStreamEOFError:
+      expect LPStreamClosedError:
         await chann.readExactly(addr data[0], 1)
 
       await conn.close()
@@ -205,7 +205,7 @@ suite "Mplex":
       let fut = chann.readExactly(addr data[0], 1)
 
       await chann.reset()
-      expect LPStreamEOFError:
+      expect LPStreamClosedError:
         await fut
 
       await conn.close()

tests/testmultiaddress.nim

@@ -22,6 +22,8 @@ const
     "/ip6zone/x/ip6/fe80::1/udp/1234/quic",
     "/onion/timaq4ygg2iegci7:1234",
     "/onion/timaq4ygg2iegci7:80/http",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:1234",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80/http",
     "/udp/0",
     "/tcp/0",
     "/sctp/0",
@@ -79,6 +81,12 @@ const
     "/onion/timaq4ygg2iegci7:-1",
     "/onion/timaq4ygg2iegci7",
     "/onion/timaq4ygg2iegci@:666",
+    "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666",
     "/udp/1234/sctp",
     "/udp/1234/udt/1234",
     "/udp/1234/utp/1234",
@@ -170,6 +178,12 @@ const
     "/onion/timaq4ygg2iegci7:-1",
     "/onion/timaq4ygg2iegci7",
     "/onion/timaq4ygg2iegci@:666",
+    "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd",
+    "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666",
     "/udp/1234/sctp",
     "/udp/1234/udt/1234",
     "/udp/1234/utp/1234",
@@ -372,7 +386,24 @@ suite "MultiAddress test suite":
     let ma = MultiAddress.init("/ip4/0.0.0.0/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio/").get()
     check:
       $ma[0..0].get() == "/ip4/0.0.0.0"
+      $ma[^1].get() == "/unix/stdio"
+      ma[-100].isErr()
+      ma[100].isErr()
+      ma[^100].isErr()
+      ma[^0].isErr()
       $ma[0..1].get() == "/ip4/0.0.0.0/tcp/0"
       $ma[1..2].get() == "/tcp/0/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"
       $ma[^3..^1].get() == "/p2p-circuit/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSuNEXT/unix/stdio"
       ma[5..7].isErr()
+
+  test "[](MultiCodec) test":
+    let onionMAStr = "/onion3/torchdeedp3i2jigzjdmfpn5ttjhthh5wbmda2rr3jvqjg5p77c54dqd:80"
+    let ma = MultiAddress.init(onionMAStr).get()
+    check $(ma[multiCodec("onion3")].tryGet()) == onionMAStr
+
+    let onionMAWithTcpStr = "/onion3/torchdeedp3i2jigzjdmfpn5ttjhthh5wbmda2rr3jvqjg5p77c54dqd:80/tcp/80"
+    let maWithTcp = MultiAddress.init(onionMAWithTcpStr).get()
+    check $(maWithTcp[multiCodec("onion3")].tryGet()) == onionMAStr
+
+
+

tests/testnameresolve.nim

@@ -139,7 +139,18 @@ suite "Name resolving":
     asyncTest "dnsaddr infinite recursion":
       resolver.txtResponses["_dnsaddr.bootstrap.libp2p.io"] = @["dnsaddr=/dnsaddr/bootstrap.libp2p.io"]
 
-      check testOne("/dnsaddr/bootstrap.libp2p.io/", "/dnsaddr/bootstrap.libp2p.io/")
+      check testOne("/dnsaddr/bootstrap.libp2p.io/", newSeq[string]())
+
+    test "getHostname":
+      check:
+        MultiAddress.init("/dnsaddr/bootstrap.libp2p.io/").tryGet().getHostname == "bootstrap.libp2p.io"
+        MultiAddress.init("").tryGet().getHostname == ""
+        MultiAddress.init("/ip4/147.75.69.143/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "147.75.69.143"
+        MultiAddress.init("/ip6/2604:1380:1000:6000::1/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN").tryGet().getHostname == "2604:1380:1000:6000::1"
+        MultiAddress.init("/dns/localhost/udp/0").tryGet().getHostname == "localhost"
+        MultiAddress.init("/dns4/hello.com/udp/0").tryGet().getHostname == "hello.com"
+        MultiAddress.init("/dns6/hello.com/udp/0").tryGet().getHostname == "hello.com"
+        MultiAddress.init("/wss/").tryGet().getHostname == ""
 
   suite "DNS Resolving":
     teardown:

tests/testnative.nim

@@ -37,5 +37,7 @@ import testtcptransport,
        testmplex,
        testrelayv1,
        testrelayv2,
+       testrendezvous,
+       testdiscovery,
        testyamux,
        testautonat

tests/testnoise.nim

@@ -60,8 +60,7 @@ method init(p: TestProto) {.gcsafe.} =
 proc createSwitch(ma: MultiAddress; outgoing: bool, secio: bool = false): (Switch, PeerInfo) =
   var
     privateKey = PrivateKey.random(ECDSA, rng[]).get()
-    peerInfo = PeerInfo.new(privateKey)
+    peerInfo = PeerInfo.new(privateKey, @[ma])
-  peerInfo.addrs.add(ma)
 
   proc createMplex(conn: Connection): Muxer =
     result = Mplex.new(conn)
@@ -104,7 +103,7 @@ suite "Noise":
 
     proc acceptHandler() {.async.} =
       let conn = await transport1.accept()
-      let sconn = await serverNoise.secure(conn, false)
+      let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId))
       try:
         await sconn.write("Hello!")
       finally:
@@ -119,8 +118,7 @@ suite "Noise":
       clientNoise = Noise.new(rng, clientPrivKey, outgoing = true)
       conn = await transport2.dial(transport1.addrs[0])
 
-    conn.peerId = serverInfo.peerId
+    let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId))
-    let sconn = await clientNoise.secure(conn, true)
 
     var msg = newSeq[byte](6)
     await sconn.readExactly(addr msg[0], 6)
@@ -149,7 +147,7 @@ suite "Noise":
       var conn: Connection
       try:
         conn = await transport1.accept()
-        discard await serverNoise.secure(conn, false)
+        discard await serverNoise.secure(conn, false, Opt.none(PeerId))
       except CatchableError:
         discard
       finally:
@@ -162,11 +160,10 @@ suite "Noise":
       clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs)
       clientNoise = Noise.new(rng, clientPrivKey, outgoing = true, commonPrologue = @[1'u8, 2'u8, 3'u8])
       conn = await transport2.dial(transport1.addrs[0])
-    conn.peerId = serverInfo.peerId
 
     var sconn: Connection = nil
     expect(NoiseDecryptTagError):
-      sconn = await clientNoise.secure(conn, true)
+      sconn = await clientNoise.secure(conn, true, Opt.some(conn.peerId))
 
     await conn.close()
     await handlerWait
@@ -186,7 +183,7 @@ suite "Noise":
 
     proc acceptHandler() {.async, gcsafe.} =
       let conn = await transport1.accept()
-      let sconn = await serverNoise.secure(conn, false)
+      let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId))
       defer:
         await sconn.close()
         await conn.close()
@@ -202,8 +199,7 @@ suite "Noise":
       clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs)
       clientNoise = Noise.new(rng, clientPrivKey, outgoing = true)
       conn = await transport2.dial(transport1.addrs[0])
-    conn.peerId = serverInfo.peerId
+    let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId))
-    let sconn = await clientNoise.secure(conn, true)
 
     await sconn.write("Hello!")
     await acceptFut
@@ -230,7 +226,7 @@ suite "Noise":
 
     proc acceptHandler() {.async, gcsafe.} =
       let conn = await transport1.accept()
-      let sconn = await serverNoise.secure(conn, false)
+      let sconn = await serverNoise.secure(conn, false, Opt.none(PeerId))
       defer:
         await sconn.close()
       let msg = await sconn.readLp(1024*1024)
@@ -244,8 +240,7 @@ suite "Noise":
       clientInfo = PeerInfo.new(clientPrivKey, transport1.addrs)
       clientNoise = Noise.new(rng, clientPrivKey, outgoing = true)
       conn = await transport2.dial(transport1.addrs[0])
-    conn.peerId = serverInfo.peerId
+    let sconn = await clientNoise.secure(conn, true, Opt.some(serverInfo.peerId))
-    let sconn = await clientNoise.secure(conn, true)
 
     await sconn.writeLp(hugePayload)
     await readTask
@@ -299,7 +294,7 @@ suite "Noise":
     (switch2, peerInfo2) = createSwitch(ma2, true, true) # secio, we want to fail
     await switch1.start()
     await switch2.start()
-    expect(UpgradeFailedError):
+    expect(DialFailedError):
       let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
 
     await allFuturesThrowing(

tests/testpeerinfo.nim

@@ -30,6 +30,8 @@ suite "PeerInfo":
       multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()]
       peerInfo = PeerInfo.new(seckey, multiAddresses)
 
+    waitFor(peerInfo.update())
+
     let
       env = peerInfo.signedPeerRecord.envelope
       rec = PeerRecord.decode(env.payload()).tryGet()
@@ -47,3 +49,17 @@ suite "PeerInfo":
       rec.addresses.len == 2
       rec.addresses[0].address == multiAddresses[0]
       rec.addresses[1].address == multiAddresses[1]
+
+  test "Public address mapping":
+    let
+      seckey = PrivateKey.random(ECDSA, rng[]).get()
+      multiAddresses = @[MultiAddress.init("/ip4/0.0.0.0/tcp/24").tryGet(), MultiAddress.init("/ip4/0.0.0.0/tcp/25").tryGet()]
+      multiAddresses2 = @[MultiAddress.init("/ip4/8.8.8.8/tcp/33").tryGet()]
+
+    proc addressMapper(input: seq[MultiAddress]): Future[seq[MultiAddress]] {.async.} =
+      check input == multiAddresses
+      await sleepAsync(0.seconds)
+      return multiAddresses2
+    var peerInfo = PeerInfo.new(seckey, multiAddresses, addressMappers = @[addressMapper])
+    waitFor peerInfo.update()
+    check peerInfo.addrs == multiAddresses2

tests/testrelayv1.nim

@@ -2,11 +2,11 @@
 
 import options, bearssl, chronos
 import stew/byteutils
-import ../libp2p/[protocols/relay/relay,
+import ../libp2p/[protocols/connectivity/relay/relay,
-                  protocols/relay/client,
+                  protocols/connectivity/relay/client,
-                  protocols/relay/messages,
+                  protocols/connectivity/relay/messages,
-                  protocols/relay/utils,
+                  protocols/connectivity/relay/utils,
-                  protocols/relay/rtransport,
+                  protocols/connectivity/relay/rtransport,
                   multiaddress,
                   peerinfo,
                   peerid,

tests/testrelayv2.nim

@@ -2,10 +2,10 @@
 
 import bearssl, chronos, options
 import ../libp2p
-import ../libp2p/[protocols/relay/relay,
+import ../libp2p/[protocols/connectivity/relay/relay,
-                  protocols/relay/messages,
+                  protocols/connectivity/relay/messages,
-                  protocols/relay/utils,
+                  protocols/connectivity/relay/utils,
-                  protocols/relay/client]
+                  protocols/connectivity/relay/client]
 import ./helpers
 import std/times
 import stew/byteutils
@@ -118,7 +118,6 @@ suite "Circuit Relay V2":
     asyncTeardown:
       checkTrackers()
     var
-      addrs {.threadvar.}: MultiAddress
       customProtoCodec {.threadvar.}: string
       proto {.threadvar.}: LPProtocol
       ttl {.threadvar.}: int
@@ -145,9 +144,6 @@ suite "Circuit Relay V2":
       src = createSwitch(srcCl)
       dst = createSwitch(dstCl)
       rel = newStandardSwitch()
-      addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
-                                $dst.peerInfo.peerId).get()
 
     asyncTest "Connection succeed":
       proto.handler = proc(conn: Connection, proto: string) {.async.} =
@@ -167,6 +163,10 @@ suite "Circuit Relay V2":
       await src.start()
       await dst.start()
 
+      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                $dst.peerInfo.peerId).get()
+
       await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
 
@@ -200,6 +200,10 @@ suite "Circuit Relay V2":
       await src.start()
       await dst.start()
 
+      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                $dst.peerInfo.peerId).get()
+
       await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
 
@@ -245,6 +249,10 @@ take to the ship.""")
       await src.start()
       await dst.start()
 
+      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                $dst.peerInfo.peerId).get()
+
       await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
 
@@ -277,6 +285,10 @@ take to the ship.""")
       await src.start()
       await dst.start()
 
+      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                $dst.peerInfo.peerId).get()
+
       await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
 
@@ -308,11 +320,6 @@ take to the ship.""")
         rel2Cl = RelayClient.new(canHop = true)
         rel2 = createSwitch(rel2Cl)
         rv2 = Relay.new()
-        addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                     $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
-                                     $rel2.peerInfo.peerId & "/p2p/" &
-                                     $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" &
-                                     $dst.peerInfo.peerId).get() ]
       rv2.setup(rel)
       rel.mount(rv2)
       dst.mount(proto)
@@ -321,6 +328,13 @@ take to the ship.""")
       await src.start()
       await dst.start()
 
+      let
+        addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                     $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                     $rel2.peerInfo.peerId & "/p2p/" &
+                                     $rel2.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                     $dst.peerInfo.peerId).get() ]
+
       await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await rel2.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
       await dst.connect(rel2.peerInfo.peerId, rel2.peerInfo.addrs)
@@ -367,6 +381,16 @@ take to the ship.""")
         switchA = createSwitch(clientA)
         switchB = createSwitch(clientB)
         switchC = createSwitch(clientC)
+
+      switchA.mount(protoBCA)
+      switchB.mount(protoCAB)
+      switchC.mount(protoABC)
+
+      await switchA.start()
+      await switchB.start()
+      await switchC.start()
+
+      let
         addrsABC = MultiAddress.init($switchB.peerInfo.addrs[0] & "/p2p/" &
                                      $switchB.peerInfo.peerId & "/p2p-circuit/p2p/" &
                                      $switchC.peerInfo.peerId).get()
@@ -376,13 +400,6 @@ take to the ship.""")
         addrsCAB = MultiAddress.init($switchA.peerInfo.addrs[0] & "/p2p/" &
                                      $switchA.peerInfo.peerId & "/p2p-circuit/p2p/" &
                                      $switchB.peerInfo.peerId).get()
-      switchA.mount(protoBCA)
-      switchB.mount(protoCAB)
-      switchC.mount(protoABC)
-
-      await switchA.start()
-      await switchB.start()
-      await switchC.start()
 
       await switchA.connect(switchB.peerInfo.peerId, switchB.peerInfo.addrs)
       await switchB.connect(switchC.peerInfo.peerId, switchC.peerInfo.addrs)

tests/testrendezvous.nim

@@ -0,0 +1,125 @@
+{.used.}
+
+import options, sequtils, strutils
+import stew/byteutils, chronos
+import ../libp2p/[protocols/rendezvous,
+                  switch,
+                  builders,]
+import ./helpers
+
+proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withRendezVous(rdv)
+    .build()
+
+suite "RendezVous":
+  teardown:
+    checkTrackers()
+  asyncTest "Simple local test":
+    let
+      rdv = RendezVous.new()
+      s = createSwitch(rdv)
+
+    await s.start()
+    let res0 = rdv.requestLocally("empty")
+    check res0.len == 0
+    await rdv.advertise("foo")
+    let res1 = rdv.requestLocally("foo")
+    check:
+      res1.len == 1
+      res1[0] == s.peerInfo.signedPeerRecord.data
+    let res2 = rdv.requestLocally("bar")
+    check res2.len == 0
+    rdv.unsubscribeLocally("foo")
+    let res3 = rdv.requestLocally("foo")
+    check res3.len == 0
+    await s.stop()
+
+  asyncTest "Simple remote test":
+    let
+      rdv = RendezVous.new()
+      client = createSwitch(rdv)
+      remoteSwitch = createSwitch()
+
+    await client.start()
+    await remoteSwitch.start()
+    await client.connect(remoteSwitch.peerInfo.peerId, remoteSwitch.peerInfo.addrs)
+    let res0 = await rdv.request("empty")
+    check res0.len == 0
+    await rdv.advertise("foo")
+    let res1 = await rdv.request("foo")
+    check:
+      res1.len == 1
+      res1[0] == client.peerInfo.signedPeerRecord.data
+    let res2 = await rdv.request("bar")
+    check res2.len == 0
+    await rdv.unsubscribe("foo")
+    let res3 = await rdv.request("foo")
+    check res3.len == 0
+    await allFutures(client.stop(), remoteSwitch.stop())
+
+  asyncTest "Harder remote test":
+    var
+      rdvSeq: seq[RendezVous] = @[]
+      clientSeq: seq[Switch] = @[]
+      remoteSwitch = createSwitch()
+
+    for x in 0..10:
+      rdvSeq.add(RendezVous.new())
+      clientSeq.add(createSwitch(rdvSeq[^1]))
+    await remoteSwitch.start()
+    await allFutures(clientSeq.mapIt(it.start()))
+    await allFutures(clientSeq.mapIt(remoteSwitch.connect(it.peerInfo.peerId, it.peerInfo.addrs)))
+    await allFutures(rdvSeq.mapIt(it.advertise("foo")))
+    var data = clientSeq.mapIt(it.peerInfo.signedPeerRecord.data)
+    let res1 = await rdvSeq[0].request("foo", 5)
+    check res1.len == 5
+    for d in res1:
+      check d in data
+    data.keepItIf(it notin res1)
+    let res2 = await rdvSeq[0].request("foo")
+    check res2.len == 5
+    for d in res2:
+      check d in data
+    let res3 = await rdvSeq[0].request("foo")
+    check res3.len == 0
+    await remoteSwitch.stop()
+    await allFutures(clientSeq.mapIt(it.stop()))
+
+  asyncTest "Simple cookie test":
+    let
+      rdvA = RendezVous.new()
+      rdvB = RendezVous.new()
+      clientA = createSwitch(rdvA)
+      clientB = createSwitch(rdvB)
+      remoteSwitch = createSwitch()
+
+    await clientA.start()
+    await clientB.start()
+    await remoteSwitch.start()
+    await clientA.connect(remoteSwitch.peerInfo.peerId, remoteSwitch.peerInfo.addrs)
+    await clientB.connect(remoteSwitch.peerInfo.peerId, remoteSwitch.peerInfo.addrs)
+    await rdvA.advertise("foo")
+    let res1 = await rdvA.request("foo")
+    await rdvB.advertise("foo")
+    let res2 = await rdvA.request("foo")
+    check:
+      res2.len == 1
+      res2[0] == clientB.peerInfo.signedPeerRecord.data
+    await allFutures(clientA.stop(), clientB.stop(), remoteSwitch.stop())
+
+  asyncTest "Various local error":
+    let
+      rdv = RendezVous.new()
+      switch = createSwitch(rdv)
+    expect RendezVousError: discard await rdv.request("A".repeat(300))
+    expect RendezVousError: discard await rdv.request("A", -1)
+    expect RendezVousError: discard await rdv.request("A", 3000)
+    expect RendezVousError: await rdv.advertise("A".repeat(300))
+    expect RendezVousError: await rdv.advertise("A", 2.weeks)
+    expect RendezVousError: await rdv.advertise("A", 5.minutes)

tests/testswitch.nim

@@ -201,6 +201,32 @@ suite "Switch":
     check not switch1.isConnected(switch2.peerInfo.peerId)
     check not switch2.isConnected(switch1.peerInfo.peerId)
 
+  asyncTest "e2e connect to peer with unknown PeerId":
+    let resolver = MockResolver.new()
+    let switch1 = newStandardSwitch(secureManagers = [SecureProtocol.Noise])
+    let switch2 = newStandardSwitch(secureManagers = [SecureProtocol.Noise], nameResolver = resolver)
+    await switch1.start()
+    await switch2.start()
+
+    # via dnsaddr
+    resolver.txtResponses["_dnsaddr.test.io"] = @[
+      "dnsaddr=" & $switch1.peerInfo.addrs[0] & "/p2p/" & $switch1.peerInfo.peerId,
+    ]
+
+    check: (await switch2.connect(@[MultiAddress.init("/dnsaddr/test.io/").tryGet()])) == switch1.peerInfo.peerId
+    await switch2.disconnect(switch1.peerInfo.peerId)
+
+    # via direct ip
+    check not switch2.isConnected(switch1.peerInfo.peerId)
+    check: (await switch2.connect(switch1.peerInfo.addrs)) == switch1.peerInfo.peerId
+
+    await switch2.disconnect(switch1.peerInfo.peerId)
+
+    await allFuturesThrowing(
+      switch1.stop(),
+      switch2.stop()
+    )
+
   asyncTest "e2e should not leak on peer disconnect":
     let switch1 = newStandardSwitch()
     let switch2 = newStandardSwitch()
@@ -651,7 +677,7 @@ suite "Switch":
     await switch.start()
 
     var peerId = PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).get()
-    expect LPStreamClosedError, LPStreamEOFError:
+    expect DialFailedError:
       await switch.connect(peerId, transport.addrs)
 
     await handlerWait
@@ -980,9 +1006,10 @@ suite "Switch":
     await srcWsSwitch.start()
 
     resolver.txtResponses["_dnsaddr.test.io"] = @[
-      "dnsaddr=" & $destSwitch.peerInfo.addrs[0],
+      "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[0][1..^1].tryGet() & "/p2p/" & $destSwitch.peerInfo.peerId,
-      "dnsaddr=" & $destSwitch.peerInfo.addrs[1]
+      "dnsaddr=/dns4/localhost" & $destSwitch.peerInfo.addrs[1][1..^1].tryGet()
     ]
+    resolver.ipResponses[("localhost", false)] = @["127.0.0.1"]
 
     let testAddr = MultiAddress.init("/dnsaddr/test.io/").tryGet()
 
@@ -1021,3 +1048,12 @@ suite "Switch":
     await conn.close()
     await src.stop()
     await dst.stop()
+
+  asyncTest "switch failing to start stops properly":
+    let switch = newStandardSwitch(
+      addrs = @[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet(), MultiAddress.init("/ip4/1.1.1.1/tcp/0").tryGet()]
+    )
+
+    expect LPError:
+      await switch.start()
+    # test is that this doesn't leak

tests/testyamux.nim

@@ -152,3 +152,36 @@ suite "Yamux":
         expect(LPStreamEOFError): await wrFut[i]
       writerBlocker.complete()
       await streamA.close()
+
+  suite "Exception testing":
+    asyncTest "Local & Remote close":
+      mSetup()
+
+      yamuxb.streamHandler = proc(conn: Connection) {.async.} =
+        check (await conn.readLp(100)) == fromHex("1234")
+        await conn.close()
+        expect LPStreamClosedError: await conn.writeLp(fromHex("102030"))
+        check (await conn.readLp(100)) == fromHex("5678")
+
+      let streamA = await yamuxa.newStream()
+      await streamA.writeLp(fromHex("1234"))
+      expect LPStreamRemoteClosedError: discard await streamA.readLp(100)
+      await streamA.writeLp(fromHex("5678"))
+      await streamA.close()
+
+    asyncTest "Local & Remote reset":
+      mSetup()
+      let blocker = newFuture[void]()
+
+      yamuxb.streamHandler = proc(conn: Connection) {.async.} =
+        await blocker
+        expect LPStreamResetError: discard await conn.readLp(100)
+        expect LPStreamResetError: await conn.writeLp(fromHex("1234"))
+        await conn.close()
+
+      let streamA = await yamuxa.newStream()
+      await yamuxa.close()
+      expect LPStreamClosedError: await streamA.writeLp(fromHex("1234"))
+      expect LPStreamClosedError: discard await streamA.readLp(100)
+      blocker.complete()
+      await streamA.close()

tools/markdown_builder.nim

@@ -0,0 +1,29 @@
+import os, strutils
+
+let contents =
+  if paramCount() > 0:
+    readFile(paramStr(1))
+  else:
+    stdin.readAll()
+
+var code = ""
+for line in contents.splitLines(true):
+  let
+    stripped = line.strip()
+    isMarkdown = stripped.startsWith("##")
+
+  if isMarkdown:
+    if code.strip.len > 0:
+      echo "```nim"
+      echo code.strip(leading = false)
+      echo "```"
+      code = ""
+    echo(if stripped.len > 3: stripped[3..^1]
+         else: "")
+  else:
+    code &= line
+if code.strip.len > 0:
+  echo ""
+  echo "```nim"
+  echo code
+  echo "```"