From 843d32f8dbb3b12df8eee6b29a386ed66b001bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 5 Aug 2020 01:27:59 +0200 Subject: [PATCH] put expensive metrics under a Nim define (#310) --- README.md | 49 +++++++++++++++----------- libp2p/muxers/mplex/mplex.nim | 21 ++++++----- libp2p/protocols/pubsub/floodsub.nim | 3 +- libp2p/protocols/pubsub/gossipsub.nim | 5 +-- libp2p/protocols/pubsub/pubsub.nim | 3 +- libp2p/protocols/pubsub/pubsubpeer.nim | 33 +++++++++-------- 6 files changed, 67 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index d0877d550..7a85e1e73 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wr ## Project Status The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :) -Check our [examples folder](/examples) to get started! +Check our [examples folder](/examples) to get started! # Table of Contents - [Background](#background) @@ -38,24 +38,24 @@ Check our [examples folder](/examples) to get started! - [Tests](#tests) - [Packages](#packages) - [Contribute](#contribute) - - [Core Developers](#core-developers) + - [Core Developers](#core-developers) - [License](#license) -## Background +## Background libp2p is a networking stack and library modularized out of [The IPFS Project](https://github.com/ipfs/ipfs), and bundled separately for other tools to use. -libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability. +libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability. libp2p grew out of IPFS, but it is built so that lots of people can use it, for lots of different projects. -- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io). -- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages. +- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io). +- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages. ## Install ``` nimble install libp2p ``` -### Prerequisite +### Prerequisite - [Nim](https://nim-lang.org/install.html) ## Usage @@ -64,18 +64,19 @@ nimble install libp2p The specification is available in the [docs/api](docs/api) folder. ### Getting Started -Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide. +Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide. -### Tutorials and Examples +### Tutorials and Examples Example code can be found in the [examples folder](/examples). + #### Direct Chat Tutorial - [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO. - [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands. -- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node. +- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node. ### Using the Go Daemon -Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md). +Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md). Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon); @@ -88,10 +89,10 @@ cd nim-libp2p nimble install ``` -### Tests +### Tests #### Prerequisite -- [Go 1.12+](https://golang.org/dl/) +- [Go 1.12+](https://golang.org/dl/) #### Run unit tests ```sh @@ -99,7 +100,7 @@ nimble install nimble test ``` -### Packages +### Packages List of packages currently in existence for nim-libp2p: @@ -131,7 +132,7 @@ List of packages currently in existence for nim-libp2p: - [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim) -Packages that exist in the original libp2p specs and are under active development: +Packages that exist in the original libp2p specs and are under active development: - libp2p-daemon - libp2p-webrtc-direct - libp2p-webrtc-star @@ -146,16 +147,23 @@ Packages that exist in the original libp2p specs and are under active developmen - libp2p-nat-mgnr - libp2p-utils -** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation. +** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation. +### Tips and tricks + +- enable expensive metrics: + +```bash +nim c -d:libp2p_expensive_metrics some_file.nim +``` ## Contribute The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can: -- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it. -- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs. +- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it. +- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs. - **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests). -### Core Developers +### Core Developers [@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Giovanni Petrantoni](https://github.com/sinkingsugar), [Zahary Karadjov](https://github.com/zah) ## License @@ -168,4 +176,5 @@ or * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) -at your option. This file may not be copied, modified, or distributed except according to those terms. +at your option. These files may not be copied, modified, or distributed except according to those terms. + diff --git a/libp2p/muxers/mplex/mplex.nim b/libp2p/muxers/mplex/mplex.nim index 98cf462d3..214d99d2b 100644 --- a/libp2p/muxers/mplex/mplex.nim +++ b/libp2p/muxers/mplex/mplex.nim @@ -23,7 +23,8 @@ export muxer logScope: topics = "mplex" -declareGauge(libp2p_mplex_channels, "mplex channels", labels = ["initiator", "peer"]) +when defined(libp2p_expensive_metrics): + declareGauge(libp2p_mplex_channels, "mplex channels", labels = ["initiator", "peer"]) type Mplex* = ref object of Muxer @@ -76,10 +77,11 @@ proc newStreamInternal*(m: Mplex, "channel slot already taken!") m.getChannelList(initiator)[id] = result - libp2p_mplex_channels.set( - m.getChannelList(initiator).len.int64, - labelValues = [$initiator, - $m.connection.peerInfo]) + when defined(libp2p_expensive_metrics): + libp2p_mplex_channels.set( + m.getChannelList(initiator).len.int64, + labelValues = [$initiator, + $m.connection.peerInfo]) proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} = ## remove the local channel from the internal tables @@ -89,10 +91,11 @@ proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} = m.getChannelList(chann.initiator).del(chann.id) trace "cleaned up channel", id = chann.id - libp2p_mplex_channels.set( - m.getChannelList(chann.initiator).len.int64, - labelValues = [$chann.initiator, - $m.connection.peerInfo]) + when defined(libp2p_expensive_metrics): + libp2p_mplex_channels.set( + m.getChannelList(chann.initiator).len.int64, + labelValues = [$chann.initiator, + $m.connection.peerInfo]) proc handleStream(m: Mplex, chann: LPChannel) {.async.} = ## call the muxer stream handler for this channel diff --git a/libp2p/protocols/pubsub/floodsub.nim b/libp2p/protocols/pubsub/floodsub.nim index d0adce3e6..7e28d2c64 100644 --- a/libp2p/protocols/pubsub/floodsub.nim +++ b/libp2p/protocols/pubsub/floodsub.nim @@ -140,7 +140,8 @@ method publish*(f: FloodSub, # start the future but do not wait yet let published = await f.publishHelper(f.floodsub.getOrDefault(topic), @[msg], timeout) - libp2p_pubsub_messages_published.inc(labelValues = [topic]) + when defined(libp2p_expensive_metrics): + libp2p_pubsub_messages_published.inc(labelValues = [topic]) trace "published message to peers", peers = published, msg = msg.shortLog() diff --git a/libp2p/protocols/pubsub/gossipsub.nim b/libp2p/protocols/pubsub/gossipsub.nim index 9b3d0e4a5..4bf3a3d09 100644 --- a/libp2p/protocols/pubsub/gossipsub.nim +++ b/libp2p/protocols/pubsub/gossipsub.nim @@ -518,8 +518,9 @@ method publish*(g: GossipSub, g.mcache.put(msgId, msg) let published = await g.publishHelper(peers, @[msg], timeout) - if published > 0: - libp2p_pubsub_messages_published.inc(labelValues = [topic]) + when defined(libp2p_expensive_metrics): + if published > 0: + libp2p_pubsub_messages_published.inc(labelValues = [topic]) trace "published message to peers", peers = published, msg = msg.shortLog() diff --git a/libp2p/protocols/pubsub/pubsub.nim b/libp2p/protocols/pubsub/pubsub.nim index f6f6de0e0..0fd75dffd 100644 --- a/libp2p/protocols/pubsub/pubsub.nim +++ b/libp2p/protocols/pubsub/pubsub.nim @@ -27,7 +27,8 @@ declareGauge(libp2p_pubsub_peers, "pubsub peer instances") declareGauge(libp2p_pubsub_topics, "pubsub subscribed topics") declareCounter(libp2p_pubsub_validation_success, "pubsub successfully validated messages") declareCounter(libp2p_pubsub_validation_failure, "pubsub failed validated messages") -declarePublicCounter(libp2p_pubsub_messages_published, "published messages", labels = ["topic"]) +when defined(libp2p_expensive_metrics): + declarePublicCounter(libp2p_pubsub_messages_published, "published messages", labels = ["topic"]) type TopicHandler* = proc(topic: string, diff --git a/libp2p/protocols/pubsub/pubsubpeer.nim b/libp2p/protocols/pubsub/pubsubpeer.nim index 1bbfdbcd8..c215c3828 100644 --- a/libp2p/protocols/pubsub/pubsubpeer.nim +++ b/libp2p/protocols/pubsub/pubsubpeer.nim @@ -21,10 +21,11 @@ import rpc/[messages, message, protobuf], logScope: topics = "pubsubpeer" -declareCounter(libp2p_pubsub_sent_messages, "number of messages sent", labels = ["id", "topic"]) -declareCounter(libp2p_pubsub_received_messages, "number of messages received", labels = ["id", "topic"]) -declareCounter(libp2p_pubsub_skipped_received_messages, "number of received skipped messages", labels = ["id"]) -declareCounter(libp2p_pubsub_skipped_sent_messages, "number of sent skipped messages", labels = ["id"]) +when defined(libp2p_expensive_metrics): + declareCounter(libp2p_pubsub_sent_messages, "number of messages sent", labels = ["id", "topic"]) + declareCounter(libp2p_pubsub_received_messages, "number of messages received", labels = ["id", "topic"]) + declareCounter(libp2p_pubsub_skipped_received_messages, "number of received skipped messages", labels = ["id"]) + declareCounter(libp2p_pubsub_skipped_sent_messages, "number of sent skipped messages", labels = ["id"]) const DefaultReadTimeout* = 1.minutes @@ -91,7 +92,8 @@ proc handle*(p: PubSubPeer, conn: Connection) {.async.} = let digest = $(sha256.digest(data)) trace "read data from peer", data = data.shortLog if digest in p.recvdRpcCache: - libp2p_pubsub_skipped_received_messages.inc(labelValues = [p.id]) + when defined(libp2p_expensive_metrics): + libp2p_pubsub_skipped_received_messages.inc(labelValues = [p.id]) trace "message already received, skipping" continue @@ -106,10 +108,11 @@ proc handle*(p: PubSubPeer, conn: Connection) {.async.} = # trigger hooks p.recvObservers(msg) - for m in msg.messages: - for t in m.topicIDs: - # metrics - libp2p_pubsub_received_messages.inc(labelValues = [p.id, t]) + when defined(libp2p_expensive_metrics): + for m in msg.messages: + for t in m.topicIDs: + # metrics + libp2p_pubsub_received_messages.inc(labelValues = [p.id, t]) await p.handler(p, @[msg]) p.recvdRpcCache.put(digest) @@ -148,7 +151,8 @@ proc send*( let digest = $(sha256.digest(encoded)) if digest in p.sentRpcCache: trace "message already sent to peer, skipping" - libp2p_pubsub_skipped_sent_messages.inc(labelValues = [p.id]) + when defined(libp2p_expensive_metrics): + libp2p_pubsub_skipped_sent_messages.inc(labelValues = [p.id]) return proc sendToRemote() {.async.} = @@ -168,10 +172,11 @@ proc send*( p.sentRpcCache.put(digest) trace "sent pubsub message to remote" - for x in mm.messages: - for t in x.topicIDs: - # metrics - libp2p_pubsub_sent_messages.inc(labelValues = [p.id, t]) + when defined(libp2p_expensive_metrics): + for x in mm.messages: + for t in x.topicIDs: + # metrics + libp2p_pubsub_sent_messages.inc(labelValues = [p.id, t]) let sendFut = sendToRemote() try: