General chat2bridge improvements (#536)

docs/tutorial/chat2.md

@@ -84,3 +84,80 @@ message Chat2Message {
 ```
 
 where `timestamp` is the Unix timestamp of the message, `nick` is the relevant `chat2` user's selected nickname and `payload` is the actual chat message being sent. The `payload` is the byte array representation of a UTF8 encoded string.
+
+# Bridge messages between `chat2` and matterbridge
+
+To facilitate `chat2` use in a variety of contexts, a `chat2bridge` can be deployed to bridge messages between `chat2` and any protocol supported by matterbridge.
+
+## Configure and run matterbridge
+
+1. Download and install [matterbridge](https://github.com/42wim/matterbridge) and configure an instance for the protocol(s) you want to bridge to.
+Basic configuration instructions [here](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config)
+2. Configure the matterbridge API.
+This is used by the `chat2bridge` to relay `chat2` messages to and from matterbridge.
+Configuration instructions for the matterbridge API can be found [here](https://github.com/42wim/matterbridge/wiki/Api).
+The full matterbridge API specification can be found [here](https://app.swaggerhub.com/apis-docs/matterbridge/matterbridge-api/0.1.0-oas3).
+The template below shows an example of a `matterbridge.toml` configuration file for bridging `chat2` to Discord.
+Follow the matterbridge [Discord instructions](https://github.com/42wim/matterbridge/wiki/Section-Discord-%28basic%29) to configure your own `Token` and `Server`.
+```toml
+[discord.mydiscord]
+
+# You can get your token by following the instructions on
+# https://github.com/42wim/matterbridge/wiki/Discord-bot-setup.
+# If you want roles/groups mentions to be shown with names instead of ID, 
+# you'll need to give your bot the "Manage Roles" permission.
+Token="MTk4NjIyNDgzNDcdOTI1MjQ4.Cl2FMZ.ZnCjm1XVW7vRze4b7Cq4se7kKWs-abD"
+
+Server="myserver" # picked from guilds the bot is connected to
+
+RemoteNickFormat="{NICK}@chat2: "
+
+[api.myapi]
+BindAddress="127.0.0.1:4242"
+Buffer=1000
+RemoteNickFormat="{NICK}@{PROTOCOL}"
+
+[[gateway]]
+name="gateway1"
+enable=true
+
+[[gateway.inout]]
+account="discord.mydiscord"
+channel="general"
+
+[[gateway.inout]]
+account="api.myapi"
+channel="api"
+```
+3. Run matterbridge using the configuration file created in the previous step.
+Note the API listening address and port in the matterbridge logs (configured as the `BindAddress` in the previous step).
+```
+./matterbridge -conf matterbridge.toml
+```
+```
+[0000]  INFO api:          Listening on 127.0.0.1:4242
+```
+## Configure and run `chat2bridge`
+1. From the `nim-waku` project directory, make the `chat2bridge` target
+```
+make chat2bridge
+```
+2. Run `chat2bridge` with the following configuration options:
+```
+--mb-host-address         Listening address of the Matterbridge host
+--mb-host-port            Listening port of the Matterbridge host
+--mb-gateway              Matterbridge gateway
+```
+```
+./build/chat2bridge --mb-host-address=127.0.0.1 --mb-host-port=4242 --mb-gateway="gateway1"
+```
+Note that `chat2bridge` encompasses a full `wakunode2` which can be configured with the normal configuration parameters.
+For a full list of configuration options, run `--help`.
+```
+./build/chat2bridge --help
+```
+## Connect `chat2bridge` to a `chat2` network
+1. To bridge messages on an existing `chat2` network, connect to any relay peer(s) in that network from `chat2bridge`.
+This can be done by either specifying the peer(s) as a `--staticnode` when starting the `chat2bridge` or calling the [`post_waku_v2_admin_v1_peers`](https://rfc.vac.dev/spec/16/#post_waku_v2_admin_v1_peers) method on the JSON-RPC API.
+Note that the latter requires the `chat2bridge` to be run with `--rpc=true` and `--rpc-admin=true`.
+1. To bridge from a new `chat2` instance, simply specify the `chat2bridge` listening address as a `chat2` [static peer](#Specifying-a-static-peer).

examples/v2/matterbridge/chat2bridge.nim

@@ -1,5 +1,7 @@
+{.push raises: [Defect, Exception].}
+
 import
-  std/[tables, times, strutils],
+  std/[tables, times, strutils, hashes, sequtils],
   chronos, confutils, chronicles, chronicles/topics_registry, metrics,
   stew/[byteutils, endians2],
   stew/shims/net as stewNet, json_rpc/rpcserver,
@@ -15,6 +17,7 @@ import
   ./config_chat2bridge
 
 declarePublicCounter chat2_mb_transfers, "Number of messages transferred between chat2 and Matterbridge", ["type"]
+declarePublicCounter chat2_mb_dropped, "Number of messages dropped", ["reason"]
 
 logScope:
   topics = "chat2bridge"
@@ -26,6 +29,7 @@ logScope:
 const
   DefaultTopic* = chat2.DefaultTopic
   DefaultContentTopic* = chat2.DefaultContentTopic
+  DeduplQSize = 20  # Maximum number of seen messages to keep in deduplication queue
 
 #########
 # Types #
@@ -37,6 +41,7 @@ type
     nodev2*: WakuNode
     running: bool
     pollPeriod: chronos.Duration
+    seen: seq[Hash] #FIFO queue
   
   MbMessageHandler* = proc (jsonNode: JsonNode) {.gcsafe.}
 
@@ -44,6 +49,18 @@ type
 # Helper funtions #
 ###################S
 
+proc containsOrAdd(sequence: var seq[Hash], hash: Hash): bool =
+  if sequence.contains(hash):
+    return true 
+
+  if sequence.len >= DeduplQSize:
+    trace "Deduplication queue full. Removing oldest item."
+    sequence.delete 0, 0  # Remove first item in queue
+  
+  sequence.add(hash)
+
+  return false
+
 proc toWakuMessage(jsonNode: JsonNode): WakuMessage =
   # Translates a Matterbridge API JSON response to a Waku v2 message
   let msgFields = jsonNode.getFields()
@@ -59,29 +76,49 @@ proc toWakuMessage(jsonNode: JsonNode): WakuMessage =
               version: 0)
 
 proc toChat2(cmb: Chat2MatterBridge, jsonNode: JsonNode) {.async.} =
-  chat2_mb_transfers.inc(labelValues = ["v1_to_v2"])
+  let msg = jsonNode.toWakuMessage()
+
+  if cmb.seen.containsOrAdd(msg.payload.hash()):
+    # This is a duplicate message. Return.
+    chat2_mb_dropped.inc(labelValues = ["duplicate"])
+    return
 
   trace "Post Matterbridge message to chat2"
+  
+  chat2_mb_transfers.inc(labelValues = ["mb_to_chat2"])
 
-  await cmb.nodev2.publish(DefaultTopic, jsonNode.toWakuMessage())
+  await cmb.nodev2.publish(DefaultTopic, msg)
 
 proc toMatterbridge(cmb: Chat2MatterBridge, msg: WakuMessage) {.gcsafe.} =
-  chat2_mb_transfers.inc(labelValues = ["v2_to_v1"])
+  if cmb.seen.containsOrAdd(msg.payload.hash()):
+    # This is a duplicate message. Return.
+    chat2_mb_dropped.inc(labelValues = ["duplicate"])
+    return
 
   trace "Post chat2 message to Matterbridge"
 
+  chat2_mb_transfers.inc(labelValues = ["chat2_to_mb"])
+
   let chat2Msg = Chat2Message.init(msg.payload)
 
   assert chat2Msg.isOk
 
-  cmb.mbClient.postMessage(text = string.fromBytes(chat2Msg[].payload),
+  try:
-                           username = chat2Msg[].nick)
+    cmb.mbClient.postMessage(text = string.fromBytes(chat2Msg[].payload),
+                             username = chat2Msg[].nick)
+  except OSError, IOError:
+    chat2_mb_dropped.inc(labelValues = ["duplicate"])
+    error "Matterbridge host unreachable. Dropping message."
 
 proc pollMatterbridge(cmb: Chat2MatterBridge, handler: MbMessageHandler) {.async.} =
   while cmb.running:
-    for jsonNode in cmb.mbClient.getMessages():
+    try:
-      handler(jsonNode)
+      for jsonNode in cmb.mbClient.getMessages():
-    
+        handler(jsonNode)
+    except OSError, IOError:
+      error "Matterbridge host unreachable. Sleeping before retrying."
+      await sleepAsync(chronos.seconds(10))
+
     await sleepAsync(cmb.pollPeriod)
 
 ##############
@@ -99,6 +136,15 @@ proc new*(T: type Chat2MatterBridge,
   # Setup Matterbridge 
   let
     mbClient = MatterbridgeClient.new(mbHostUri, mbGateway)
+  
+  # Let's verify the Matterbridge configuration before continuing
+  try:
+    if mbClient.isHealthy():
+      info "Reached Matterbridge host", host=mbClient.host
+    else:
+      raise newException(ValueError, "Matterbridge client not healthy")
+  except OSError, IOError:
+    raise newException(ValueError, "Matterbridge host unreachable")
 
   # Setup Waku v2 node
   let
@@ -185,7 +231,7 @@ when isMainModule:
 
   let
     bridge = Chat2Matterbridge.new(
-                            mbHostUri = conf.mbHostUri,
+                            mbHostUri = "http://" & $initTAddress(conf.mbHostAddress, Port(conf.mbHostPort)),
                             mbGateway = conf.mbGateway,
                             nodev2Key = conf.nodeKeyv2,
                             nodev2BindIp = conf.listenAddress, nodev2BindPort = Port(uint16(conf.libp2pTcpPort) + conf.portsShift),

examples/v2/matterbridge/config_chat2bridge.nim

@@ -105,11 +105,16 @@ type
       defaultValue: ""
       name: "filternode" }: string
     
-    # Matterbridge options
+    # Matterbridge options    
-    mbHostUri* {.
+    mbHostAddress* {.
-      desc: "Matterbridge host API address"
+      desc: "Listening address of the Matterbridge host",
-      defaultValue: "http://127.0.0.1:4242"
+      defaultValue: ValidIpAddress.init("127.0.0.1")
-      name: "mb-host-uri" }: string
+      name: "mb-host-address" }: ValidIpAddress
+
+    mbHostPort* {.
+      desc: "Listening port of the Matterbridge host",
+      defaultValue: 4242
+      name: "mb-host-port" }: uint16
     
     mbGateway* {.
       desc: "Matterbridge gateway"