From eb786607023e0773f0d2919d68716034a394b052 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 28 Sep 2022 10:40:53 +0200 Subject: [PATCH] Docs rework (#776) --- .github/workflows/doc.yml | 8 ++- .gitignore | 1 + examples/README.md | 2 +- examples/circuitrelay.nim | 12 +++- examples/directchat.nim | 2 +- examples/helloworld.nim | 2 +- examples/tutorial_1_connect.md | 108 ---------------------------- examples/tutorial_1_connect.nim | 95 ++++++++++++++++++++++++ examples/tutorial_2_customproto.md | 82 --------------------- examples/tutorial_2_customproto.nim | 73 +++++++++++++++++++ libp2p.nimble | 18 +++-- mkdocs.yml | 9 ++- tools/markdown_builder.nim | 29 ++++++++ 13 files changed, 235 insertions(+), 206 deletions(-) delete mode 100644 examples/tutorial_1_connect.md create mode 100644 examples/tutorial_1_connect.nim delete mode 100644 examples/tutorial_2_customproto.md create mode 100644 examples/tutorial_2_customproto.nim create mode 100644 tools/markdown_builder.nim diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 862c571..dce8676 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -63,7 +63,7 @@ jobs: git push origin gh-pages update_site: - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs' name: 'Rebuild website' runs-on: ubuntu-latest steps: @@ -74,8 +74,12 @@ jobs: with: python-version: 3.x + - uses: jiro4989/setup-nim-action@v1 + with: + nim-version: 'stable' + - name: Generate website - run: pip install mkdocs-material && mkdocs build + run: pip install mkdocs-material && nimble website - name: Clone the gh-pages branch uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index ec585b0..f93a08d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,6 @@ build/ .vscode/ .DS_Store tests/pubsub/testgossipsub +examples/*.md nimble.develop nimble.paths diff --git a/examples/README.md b/examples/README.md index 18b6ca8..0760073 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,5 +2,5 @@ Welcome to the nim-libp2p documentation! -Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and +Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html). diff --git a/examples/circuitrelay.nim b/examples/circuitrelay.nim index b7c66d2..bf90d97 100644 --- a/examples/circuitrelay.nim +++ b/examples/circuitrelay.nim @@ -1,6 +1,14 @@ +## # Circuit Relay example +## +## Circuit Relay can be used when a node cannot reach another node +## directly, but can reach it through a another node (the Relay). +## +## That may happen because of NAT, Firewalls, or incompatible transports. +## +## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/). import chronos, stew/byteutils -import ../libp2p, - ../libp2p/protocols/connectivity/relay/[relay, client] +import libp2p, + libp2p/protocols/connectivity/relay/[relay, client] # Helper to create a circuit relay node proc createCircuitRelaySwitch(r: Relay): Switch = diff --git a/examples/directchat.nim b/examples/directchat.nim index 9e7d99c..b550d48 100644 --- a/examples/directchat.nim +++ b/examples/directchat.nim @@ -5,7 +5,7 @@ import strformat, strutils, stew/byteutils, chronos, - ../libp2p + libp2p const DefaultAddr = "/ip4/127.0.0.1/tcp/0" diff --git a/examples/helloworld.nim b/examples/helloworld.nim index 9f5f1cb..7f144aa 100644 --- a/examples/helloworld.nim +++ b/examples/helloworld.nim @@ -1,6 +1,6 @@ import chronos # an efficient library for async import stew/byteutils # various utils -import ../libp2p # when installed through nimble, just use `import libp2p` +import libp2p ## # Create our custom protocol diff --git a/examples/tutorial_1_connect.md b/examples/tutorial_1_connect.md deleted file mode 100644 index 0dd6543..0000000 --- a/examples/tutorial_1_connect.md +++ /dev/null @@ -1,108 +0,0 @@ -# Simple ping tutorial - -Hi all, welcome to the first nim-libp2p tutorial! - -!!! tips "" - This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. - -To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. - -Hope you'll find it helpful in your journey of learning. Happy coding! ;) - -## Before you start -The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). - -Install Nim via their [official website](https://nim-lang.org/install.html). -Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. - -You can now install the latest version of `nim-libp2p`: -```bash -nimble install libp2p@#master -``` - -## A simple ping application -We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. - -!!! tips "" - You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder! - -Let's create a `part1.nim`, and import our dependencies: -```nim -import chronos - -import libp2p -import libp2p/protocols/ping -``` -[chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` - -Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: -```nim -proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = - var switch = SwitchBuilder - .new() - .withRng(rng) # Give the application RNG - .withAddress(ma) # Our local address(es) - .withTcpTransport() # Use TCP as transport - .withMplex() # Use Mplex as muxer - .withNoise() # Use Noise as secure manager - .build() - - return switch -``` -This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. - -You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. - - -Let's now start to create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() - pingProtocol = Ping.new(rng=rng) -``` -We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. -The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". - -`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. - -We can now create our two switches: -```nim - let - switch1 = createSwitch(localAddress, rng) - switch2 = createSwitch(localAddress, rng) - - switch1.mount(pingProtocol) - - await switch1.start() - await switch2.start() -``` -We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. - -Now that we've started the nodes, they are listening for incoming peers. -We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. - -We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: -```nim - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) -``` -We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: -```nim - # ping the other node and echo the ping duration - echo "ping: ", await pingProtocol.ping(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() -``` - -And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: -```nim - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports - -waitFor(main()) -``` - -You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. - -In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_1_connect.nim b/examples/tutorial_1_connect.nim new file mode 100644 index 0000000..d8d8c2b --- /dev/null +++ b/examples/tutorial_1_connect.nim @@ -0,0 +1,95 @@ +## # Simple ping tutorial +## +## Hi all, welcome to the first nim-libp2p tutorial! +## +## !!! tips "" +## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed. +## +## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications. +## +## Hope you'll find it helpful in your journey of learning. Happy coding! ;) +## +## ## Before you start +## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html). +## +## Install Nim via their [official website](https://nim-lang.org/install.html). +## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`. +## +## You can now install the latest version of `nim-libp2p`: +## ```bash +## nimble install libp2p@#master +## ``` +## +## ## A simple ping application +## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol. +## +## !!! tips "" +## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder! +## +## Let's create a `part1.nim`, and import our dependencies: +import chronos + +import libp2p +import libp2p/protocols/ping + +## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p` +## +## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once: +proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch = + var switch = SwitchBuilder + .new() + .withRng(rng) # Give the application RNG + .withAddress(ma) # Our local address(es) + .withTcpTransport() # Use TCP as transport + .withMplex() # Use Mplex as muxer + .withNoise() # Use Noise as secure manager + .build() + + return switch + +## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport. +## +## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward. +## +## +## Let's now start to create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() + pingProtocol = Ping.new(rng=rng) + ## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol. + ## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available". + ## + ## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid. + ## + ## We can now create our two switches: + let + switch1 = createSwitch(localAddress, rng) + switch2 = createSwitch(localAddress, rng) + + switch1.mount(pingProtocol) + + await switch1.start() + await switch2.start() + ## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly. + ## + ## Now that we've started the nodes, they are listening for incoming peers. + ## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`. + ## + ## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**: + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec) + ## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node: + # ping the other node and echo the ping duration + echo "ping: ", await pingProtocol.ping(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + ## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure: + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +waitFor(main()) + +## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output. +## +## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol. diff --git a/examples/tutorial_2_customproto.md b/examples/tutorial_2_customproto.md deleted file mode 100644 index aa3366c..0000000 --- a/examples/tutorial_2_customproto.md +++ /dev/null @@ -1,82 +0,0 @@ -# Custom protocol in libp2p - -In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. - -We'll now look at how to create a custom protocol inside the libp2p - -Let's create a `part2.nim`, and import our dependencies: -```nim -import chronos -import stew/byteutils - -import libp2p -``` -This is similar to the first tutorial, except we don't need to import the `Ping` protocol. - -Next, we'll declare our custom protocol -```nim -const TestCodec = "/test/proto/1.0.0" - -type TestProto = ref object of LPProtocol -``` - -We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. - -A protocol generally has two part: and handling/server part, and a dialing/client part. -Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. - -Let's start with the server part: -```nim -proc new(T: typedesc[TestProto]): T = - # every incoming connections will in be handled in this closure - proc handle(conn: Connection, proto: string) {.async, gcsafe.} = - # Read up to 1024 bytes from this connection, and transform them into - # a string - echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) - # We must close the connections ourselves when we're done with it - await conn.close() - - return T(codecs: @[TestCodec], handler: handle) -``` -This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. -In our handle, we simply read a message from the connection and `echo` it. - -We can now create our client part: -```nim -proc hello(p: TestProto, conn: Connection) {.async.} = - await conn.writeLp("Hello p2p!") -``` -Again, pretty straight-forward, we just send a message on the connection. - -We can now create our main procedure: -```nim -proc main() {.async, gcsafe.} = - let - rng = newRng() - testProto = TestProto.new() - switch1 = newStandardSwitch(rng=rng) - switch2 = newStandardSwitch(rng=rng) - - switch1.mount(testProto) - - await switch1.start() - await switch2.start() - - let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) - - await testProto.hello(conn) - - # We must close the connection ourselves when we're done with it - await conn.close() - - await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports -``` - -This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p - -We can now wrap our program by calling our main proc: -```nim -waitFor(main()) -``` - -And that's it! diff --git a/examples/tutorial_2_customproto.nim b/examples/tutorial_2_customproto.nim new file mode 100644 index 0000000..2f5789b --- /dev/null +++ b/examples/tutorial_2_customproto.nim @@ -0,0 +1,73 @@ +## # Custom protocol in libp2p +## +## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`. +## +## We'll now look at how to create a custom protocol inside the libp2p +## +## Let's create a `part2.nim`, and import our dependencies: +import chronos +import stew/byteutils + +import libp2p +## This is similar to the first tutorial, except we don't need to import the `Ping` protocol. +## +## Next, we'll declare our custom protocol +const TestCodec = "/test/proto/1.0.0" + +type TestProto = ref object of LPProtocol + +## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables. +## +## A protocol generally has two part: and handling/server part, and a dialing/client part. +## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately. +## +## Let's start with the server part: + +proc new(T: typedesc[TestProto]): T = + # every incoming connections will in be handled in this closure + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + # Read up to 1024 bytes from this connection, and transform them into + # a string + echo "Got from remote - ", string.fromBytes(await conn.readLp(1024)) + # We must close the connections ourselves when we're done with it + await conn.close() + + return T(codecs: @[TestCodec], handler: handle) + +## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol. +## In our handle, we simply read a message from the connection and `echo` it. +## +## We can now create our client part: +proc hello(p: TestProto, conn: Connection) {.async.} = + await conn.writeLp("Hello p2p!") + +## Again, pretty straight-forward, we just send a message on the connection. +## +## We can now create our main procedure: +proc main() {.async, gcsafe.} = + let + rng = newRng() + testProto = TestProto.new() + switch1 = newStandardSwitch(rng=rng) + switch2 = newStandardSwitch(rng=rng) + + switch1.mount(testProto) + + await switch1.start() + await switch2.start() + + let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec) + + await testProto.hello(conn) + + # We must close the connection ourselves when we're done with it + await conn.close() + + await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports + +## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p +## +## We can now wrap our program by calling our main proc: +waitFor(main()) + +## And that's it! diff --git a/libp2p.nimble b/libp2p.nimble index fb2ef48..a1c3b04 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -32,16 +32,16 @@ proc runTest(filename: string, verify: bool = true, sign: bool = true, rmFile "tests/" & filename.toExe proc buildSample(filename: string, run = false) = - var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off " + var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " excstr.add(" examples/" & filename) exec excstr if run: exec "./examples/" & filename.toExe rmFile "examples/" & filename.toExe -proc buildTutorial(filename: string) = - discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " & - " nim --verbosity:0 --hints:off c -" +proc tutorialToMd(filename: string) = + let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim " + writeFile(filename.replace(".nim", ".md"), markdown) task testnative, "Runs libp2p native tests": runTest("testnative") @@ -86,12 +86,18 @@ task test_slim, "Runs the (slimmed down) test suite": exec "nimble testfilter" exec "nimble examples_build" +task website, "Build the website": + tutorialToMd("examples/tutorial_1_connect.nim") + tutorialToMd("examples/tutorial_2_customproto.nim") + tutorialToMd("examples/circuitrelay.nim") + exec "mkdocs build" + task examples_build, "Build the samples": buildSample("directchat") buildSample("helloworld", true) buildSample("circuitrelay", true) - buildTutorial("examples/tutorial_1_connect.md") - buildTutorial("examples/tutorial_2_customproto.md") + buildSample("tutorial_1_connect", true) + buildSample("tutorial_2_customproto", true) # pin system # while nimble lockfile diff --git a/mkdocs.yml b/mkdocs.yml index 9c4fcbb..76c6570 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,9 @@ site_name: nim-libp2p repo_url: https://github.com/status-im/nim-libp2p repo_name: status-im/nim-libp2p site_url: https://status-im.github.io/nim-libp2p/docs -edit_uri: edit/unstable/examples/ +# Can't find a way to point the edit to the .nim instead +# of the .md +edit_uri: '' docs_dir: examples @@ -40,6 +42,7 @@ theme: nav: - Introduction: README.md - Tutorials: - - 'Part I: Simple connection': tutorial_1_connect.md - - 'Part II: Custom protocol': tutorial_2_customproto.md + - 'Simple connection': tutorial_1_connect.md + - 'Create a custom protocol': tutorial_2_customproto.md + - 'Circuit Relay': circuitrelay.md - Reference: '/nim-libp2p/master/libp2p.html' diff --git a/tools/markdown_builder.nim b/tools/markdown_builder.nim new file mode 100644 index 0000000..1477c80 --- /dev/null +++ b/tools/markdown_builder.nim @@ -0,0 +1,29 @@ +import os, strutils + +let contents = + if paramCount() > 0: + readFile(paramStr(1)) + else: + stdin.readAll() + +var code = "" +for line in contents.splitLines(true): + let + stripped = line.strip() + isMarkdown = stripped.startsWith("##") + + if isMarkdown: + if code.strip.len > 0: + echo "```nim" + echo code.strip(leading = false) + echo "```" + code = "" + echo(if stripped.len > 3: stripped[3..^1] + else: "") + else: + code &= line +if code.strip.len > 0: + echo "" + echo "```nim" + echo code + echo "```"