From 01d63963855162dd5a9c0398285738c0be3414e8 Mon Sep 17 00:00:00 2001 From: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com> Date: Fri, 7 May 2021 10:05:11 +0200 Subject: [PATCH] General chat2bridge improvements (#536) --- docs/tutorial/chat2.md | 77 +++++++++++++++++++ examples/v2/matterbridge/chat2bridge.nim | 66 +++++++++++++--- .../v2/matterbridge/config_chat2bridge.nim | 15 ++-- 3 files changed, 143 insertions(+), 15 deletions(-) diff --git a/docs/tutorial/chat2.md b/docs/tutorial/chat2.md index a1a1f1f62..29780c57d 100644 --- a/docs/tutorial/chat2.md +++ b/docs/tutorial/chat2.md @@ -84,3 +84,80 @@ message Chat2Message { ``` where `timestamp` is the Unix timestamp of the message, `nick` is the relevant `chat2` user's selected nickname and `payload` is the actual chat message being sent. The `payload` is the byte array representation of a UTF8 encoded string. + +# Bridge messages between `chat2` and matterbridge + +To facilitate `chat2` use in a variety of contexts, a `chat2bridge` can be deployed to bridge messages between `chat2` and any protocol supported by matterbridge. + +## Configure and run matterbridge + +1. Download and install [matterbridge](https://github.com/42wim/matterbridge) and configure an instance for the protocol(s) you want to bridge to. +Basic configuration instructions [here](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) +2. Configure the matterbridge API. +This is used by the `chat2bridge` to relay `chat2` messages to and from matterbridge. +Configuration instructions for the matterbridge API can be found [here](https://github.com/42wim/matterbridge/wiki/Api). +The full matterbridge API specification can be found [here](https://app.swaggerhub.com/apis-docs/matterbridge/matterbridge-api/0.1.0-oas3). +The template below shows an example of a `matterbridge.toml` configuration file for bridging `chat2` to Discord. +Follow the matterbridge [Discord instructions](https://github.com/42wim/matterbridge/wiki/Section-Discord-%28basic%29) to configure your own `Token` and `Server`. +```toml +[discord.mydiscord] + +# You can get your token by following the instructions on +# https://github.com/42wim/matterbridge/wiki/Discord-bot-setup. +# If you want roles/groups mentions to be shown with names instead of ID, +# you'll need to give your bot the "Manage Roles" permission. +Token="MTk4NjIyNDgzNDcdOTI1MjQ4.Cl2FMZ.ZnCjm1XVW7vRze4b7Cq4se7kKWs-abD" + +Server="myserver" # picked from guilds the bot is connected to + +RemoteNickFormat="{NICK}@chat2: " + +[api.myapi] +BindAddress="127.0.0.1:4242" +Buffer=1000 +RemoteNickFormat="{NICK}@{PROTOCOL}" + +[[gateway]] +name="gateway1" +enable=true + +[[gateway.inout]] +account="discord.mydiscord" +channel="general" + +[[gateway.inout]] +account="api.myapi" +channel="api" +``` +3. Run matterbridge using the configuration file created in the previous step. +Note the API listening address and port in the matterbridge logs (configured as the `BindAddress` in the previous step). +``` +./matterbridge -conf matterbridge.toml +``` +``` +[0000] INFO api: Listening on 127.0.0.1:4242 +``` +## Configure and run `chat2bridge` +1. From the `nim-waku` project directory, make the `chat2bridge` target +``` +make chat2bridge +``` +2. Run `chat2bridge` with the following configuration options: +``` +--mb-host-address Listening address of the Matterbridge host +--mb-host-port Listening port of the Matterbridge host +--mb-gateway Matterbridge gateway +``` +``` +./build/chat2bridge --mb-host-address=127.0.0.1 --mb-host-port=4242 --mb-gateway="gateway1" +``` +Note that `chat2bridge` encompasses a full `wakunode2` which can be configured with the normal configuration parameters. +For a full list of configuration options, run `--help`. +``` +./build/chat2bridge --help +``` +## Connect `chat2bridge` to a `chat2` network +1. To bridge messages on an existing `chat2` network, connect to any relay peer(s) in that network from `chat2bridge`. +This can be done by either specifying the peer(s) as a `--staticnode` when starting the `chat2bridge` or calling the [`post_waku_v2_admin_v1_peers`](https://rfc.vac.dev/spec/16/#post_waku_v2_admin_v1_peers) method on the JSON-RPC API. +Note that the latter requires the `chat2bridge` to be run with `--rpc=true` and `--rpc-admin=true`. +1. To bridge from a new `chat2` instance, simply specify the `chat2bridge` listening address as a `chat2` [static peer](#Specifying-a-static-peer). diff --git a/examples/v2/matterbridge/chat2bridge.nim b/examples/v2/matterbridge/chat2bridge.nim index 343174f2d..d39b521fb 100644 --- a/examples/v2/matterbridge/chat2bridge.nim +++ b/examples/v2/matterbridge/chat2bridge.nim @@ -1,5 +1,7 @@ +{.push raises: [Defect, Exception].} + import - std/[tables, times, strutils], + std/[tables, times, strutils, hashes, sequtils], chronos, confutils, chronicles, chronicles/topics_registry, metrics, stew/[byteutils, endians2], stew/shims/net as stewNet, json_rpc/rpcserver, @@ -15,6 +17,7 @@ import ./config_chat2bridge declarePublicCounter chat2_mb_transfers, "Number of messages transferred between chat2 and Matterbridge", ["type"] +declarePublicCounter chat2_mb_dropped, "Number of messages dropped", ["reason"] logScope: topics = "chat2bridge" @@ -26,6 +29,7 @@ logScope: const DefaultTopic* = chat2.DefaultTopic DefaultContentTopic* = chat2.DefaultContentTopic + DeduplQSize = 20 # Maximum number of seen messages to keep in deduplication queue ######### # Types # @@ -37,6 +41,7 @@ type nodev2*: WakuNode running: bool pollPeriod: chronos.Duration + seen: seq[Hash] #FIFO queue MbMessageHandler* = proc (jsonNode: JsonNode) {.gcsafe.} @@ -44,6 +49,18 @@ type # Helper funtions # ###################S +proc containsOrAdd(sequence: var seq[Hash], hash: Hash): bool = + if sequence.contains(hash): + return true + + if sequence.len >= DeduplQSize: + trace "Deduplication queue full. Removing oldest item." + sequence.delete 0, 0 # Remove first item in queue + + sequence.add(hash) + + return false + proc toWakuMessage(jsonNode: JsonNode): WakuMessage = # Translates a Matterbridge API JSON response to a Waku v2 message let msgFields = jsonNode.getFields() @@ -59,29 +76,49 @@ proc toWakuMessage(jsonNode: JsonNode): WakuMessage = version: 0) proc toChat2(cmb: Chat2MatterBridge, jsonNode: JsonNode) {.async.} = - chat2_mb_transfers.inc(labelValues = ["v1_to_v2"]) + let msg = jsonNode.toWakuMessage() + + if cmb.seen.containsOrAdd(msg.payload.hash()): + # This is a duplicate message. Return. + chat2_mb_dropped.inc(labelValues = ["duplicate"]) + return trace "Post Matterbridge message to chat2" + + chat2_mb_transfers.inc(labelValues = ["mb_to_chat2"]) - await cmb.nodev2.publish(DefaultTopic, jsonNode.toWakuMessage()) + await cmb.nodev2.publish(DefaultTopic, msg) proc toMatterbridge(cmb: Chat2MatterBridge, msg: WakuMessage) {.gcsafe.} = - chat2_mb_transfers.inc(labelValues = ["v2_to_v1"]) + if cmb.seen.containsOrAdd(msg.payload.hash()): + # This is a duplicate message. Return. + chat2_mb_dropped.inc(labelValues = ["duplicate"]) + return trace "Post chat2 message to Matterbridge" + chat2_mb_transfers.inc(labelValues = ["chat2_to_mb"]) + let chat2Msg = Chat2Message.init(msg.payload) assert chat2Msg.isOk - cmb.mbClient.postMessage(text = string.fromBytes(chat2Msg[].payload), - username = chat2Msg[].nick) + try: + cmb.mbClient.postMessage(text = string.fromBytes(chat2Msg[].payload), + username = chat2Msg[].nick) + except OSError, IOError: + chat2_mb_dropped.inc(labelValues = ["duplicate"]) + error "Matterbridge host unreachable. Dropping message." proc pollMatterbridge(cmb: Chat2MatterBridge, handler: MbMessageHandler) {.async.} = while cmb.running: - for jsonNode in cmb.mbClient.getMessages(): - handler(jsonNode) - + try: + for jsonNode in cmb.mbClient.getMessages(): + handler(jsonNode) + except OSError, IOError: + error "Matterbridge host unreachable. Sleeping before retrying." + await sleepAsync(chronos.seconds(10)) + await sleepAsync(cmb.pollPeriod) ############## @@ -99,6 +136,15 @@ proc new*(T: type Chat2MatterBridge, # Setup Matterbridge let mbClient = MatterbridgeClient.new(mbHostUri, mbGateway) + + # Let's verify the Matterbridge configuration before continuing + try: + if mbClient.isHealthy(): + info "Reached Matterbridge host", host=mbClient.host + else: + raise newException(ValueError, "Matterbridge client not healthy") + except OSError, IOError: + raise newException(ValueError, "Matterbridge host unreachable") # Setup Waku v2 node let @@ -185,7 +231,7 @@ when isMainModule: let bridge = Chat2Matterbridge.new( - mbHostUri = conf.mbHostUri, + mbHostUri = "http://" & $initTAddress(conf.mbHostAddress, Port(conf.mbHostPort)), mbGateway = conf.mbGateway, nodev2Key = conf.nodeKeyv2, nodev2BindIp = conf.listenAddress, nodev2BindPort = Port(uint16(conf.libp2pTcpPort) + conf.portsShift), diff --git a/examples/v2/matterbridge/config_chat2bridge.nim b/examples/v2/matterbridge/config_chat2bridge.nim index 56e950e97..ab9bdecac 100644 --- a/examples/v2/matterbridge/config_chat2bridge.nim +++ b/examples/v2/matterbridge/config_chat2bridge.nim @@ -105,11 +105,16 @@ type defaultValue: "" name: "filternode" }: string - # Matterbridge options - mbHostUri* {. - desc: "Matterbridge host API address" - defaultValue: "http://127.0.0.1:4242" - name: "mb-host-uri" }: string + # Matterbridge options + mbHostAddress* {. + desc: "Listening address of the Matterbridge host", + defaultValue: ValidIpAddress.init("127.0.0.1") + name: "mb-host-address" }: ValidIpAddress + + mbHostPort* {. + desc: "Listening port of the Matterbridge host", + defaultValue: 4242 + name: "mb-host-port" }: uint16 mbGateway* {. desc: "Matterbridge gateway"