Use protobuf

examples/min-js-web-chat/package-lock.json

@@ -11,6 +11,7 @@
         "@testing-library/react": "^11.1.0",
         "@testing-library/user-event": "^12.1.10",
         "js-waku": "../../build/main",
+        "protons": "^2.0.1",
         "react": "^17.0.2",
         "react-dom": "^17.0.2",
         "react-scripts": "4.0.3",
@@ -13000,6 +13001,11 @@
       "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
       "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE="
     },
+    "node_modules/multiformats": {
+      "version": "9.4.3",
+      "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.4.3.tgz",
+      "integrity": "sha512-sCNjBP/NPCeQu83Mst8IQZq9+HuR7Catvk/m7CeH0r/nupsU6gM7GINf5E1HCDRxDeU+Cgda/WPmcwQhYs3dyA=="
+    },
     "node_modules/nan": {
       "version": "2.14.2",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
@@ -15507,6 +15513,22 @@
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
       "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
     },
+    "node_modules/protocol-buffers-schema": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.5.1.tgz",
+      "integrity": "sha512-YVCvdhxWNDP8/nJDyXLuM+UFsuPk4+1PB7WGPVDzm3HTHbzFLxQYeW2iZpS4mmnXrQJGBzt230t/BbEb7PrQaw=="
+    },
+    "node_modules/protons": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/protons/-/protons-2.0.1.tgz",
+      "integrity": "sha512-FlmPorLEeCEDPu+uIn0Qardgiy5XqVA4IyNTz9wb9c0e2U7BEXdRcIbx64r09o4Abtf+4B7mkTtMbsIXMxZzKw==",
+      "dependencies": {
+        "protocol-buffers-schema": "^3.3.1",
+        "signed-varint": "^2.0.1",
+        "uint8arrays": "^2.1.3",
+        "varint": "^5.0.0"
+      }
+    },
     "node_modules/proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -17447,6 +17469,14 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
       "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
     },
+    "node_modules/signed-varint": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/signed-varint/-/signed-varint-2.0.1.tgz",
+      "integrity": "sha1-UKmYnafJjCxh2tEZvJdHDvhSgSk=",
+      "dependencies": {
+        "varint": "~5.0.0"
+      }
+    },
     "node_modules/simple-swizzle": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
@@ -19096,6 +19126,14 @@
         "node": ">=4.2.0"
       }
     },
+    "node_modules/uint8arrays": {
+      "version": "2.1.8",
+      "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-2.1.8.tgz",
+      "integrity": "sha512-qpZ/B88mSea11W3LvoimtnGWIC2i3gGuXby5wBkn8jY+OFulbaQwyjpOYSyrASqgcNEvKdAkLiOwiUt5cPSdcQ==",
+      "dependencies": {
+        "multiformats": "^9.4.2"
+      }
+    },
     "node_modules/unbox-primitive": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz",
@@ -19465,6 +19503,11 @@
         "spdx-expression-parse": "^3.0.0"
       }
     },
+    "node_modules/varint": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz",
+      "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow=="
+    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@@ -31476,6 +31519,11 @@
       "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
       "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE="
     },
+    "multiformats": {
+      "version": "9.4.3",
+      "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.4.3.tgz",
+      "integrity": "sha512-sCNjBP/NPCeQu83Mst8IQZq9+HuR7Catvk/m7CeH0r/nupsU6gM7GINf5E1HCDRxDeU+Cgda/WPmcwQhYs3dyA=="
+    },
     "nan": {
       "version": "2.14.2",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
@@ -33490,6 +33538,22 @@
         }
       }
     },
+    "protocol-buffers-schema": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.5.1.tgz",
+      "integrity": "sha512-YVCvdhxWNDP8/nJDyXLuM+UFsuPk4+1PB7WGPVDzm3HTHbzFLxQYeW2iZpS4mmnXrQJGBzt230t/BbEb7PrQaw=="
+    },
+    "protons": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/protons/-/protons-2.0.1.tgz",
+      "integrity": "sha512-FlmPorLEeCEDPu+uIn0Qardgiy5XqVA4IyNTz9wb9c0e2U7BEXdRcIbx64r09o4Abtf+4B7mkTtMbsIXMxZzKw==",
+      "requires": {
+        "protocol-buffers-schema": "^3.3.1",
+        "signed-varint": "^2.0.1",
+        "uint8arrays": "^2.1.3",
+        "varint": "^5.0.0"
+      }
+    },
     "proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -35017,6 +35081,14 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
       "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
     },
+    "signed-varint": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/signed-varint/-/signed-varint-2.0.1.tgz",
+      "integrity": "sha1-UKmYnafJjCxh2tEZvJdHDvhSgSk=",
+      "requires": {
+        "varint": "~5.0.0"
+      }
+    },
     "simple-swizzle": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
@@ -36352,6 +36424,14 @@
       "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==",
       "peer": true
     },
+    "uint8arrays": {
+      "version": "2.1.8",
+      "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-2.1.8.tgz",
+      "integrity": "sha512-qpZ/B88mSea11W3LvoimtnGWIC2i3gGuXby5wBkn8jY+OFulbaQwyjpOYSyrASqgcNEvKdAkLiOwiUt5cPSdcQ==",
+      "requires": {
+        "multiformats": "^9.4.2"
+      }
+    },
     "unbox-primitive": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz",
@@ -36646,6 +36726,11 @@
         "spdx-expression-parse": "^3.0.0"
       }
     },
+    "varint": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz",
+      "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow=="
+    },
     "vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

examples/min-js-web-chat/package.json

@@ -7,6 +7,7 @@
     "@testing-library/react": "^11.1.0",
     "@testing-library/user-event": "^12.1.10",
     "js-waku": "../../build/main",
+    "protons": "^2.0.1",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",

examples/min-js-web-chat/src/App.js

@@ -1,6 +1,7 @@
 import './App.css';
 import { getStatusFleetNodes, Waku, WakuMessage } from 'js-waku';
 import * as React from 'react';
+import protons from 'protons';
 
 const ContentTopic = `/relay-guide/1/chat/proto`;
 
@@ -8,6 +9,13 @@ const initialMessageState = {
   messages: [],
 };
 
+const proto = protons(`
+message SimpleChatMessage {
+  float timestamp = 1;
+  string text = 2;
+}
+`);
+
 function App() {
   const [waku, setWaku] = React.useState(undefined);
   const [wakuStatus, setWakuStatus] = React.useState('NotStarted');
@@ -33,7 +41,19 @@ function App() {
 
   // Need to keep the same reference around to add and delete from relay observer
   const processIncomingMessage = React.useCallback((wakuMessage) => {
-    dispatchMessages({ type: 'Add', message: wakuMessage.payloadAsUtf8 });
+    if (!wakuMessage.payload) return;
+
+    const { timestamp, text } = proto.SimpleChatMessage.decode(
+      wakuMessage.payload
+    );
+
+    dispatchMessages({
+      type: 'Add',
+      message: {
+        timestamp: new Date(timestamp),
+        text,
+      },
+    });
   }, []);
 
   React.useEffect(() => {
@@ -63,7 +83,13 @@ function App() {
         </button>
         <ul>
           {messagesState.messages.map((msg) => {
-            return <li>{msg}</li>;
+            return (
+              <li>
+                <p>
+                  {msg.timestamp.toString()}: {msg.text}
+                </p>
+              </li>
+            );
           })}
         </ul>
       </header>
@@ -79,7 +105,12 @@ async function bootstrapWaku(waku) {
 }
 
 async function sendMessage(message, waku) {
-  const wakuMessage = await WakuMessage.fromUtf8String(message, ContentTopic);
+  const payload = proto.SimpleChatMessage.encode({
+    timestamp: Date.now(),
+    text: message,
+  });
+
+  const wakuMessage = await WakuMessage.fromBytes(payload, ContentTopic);
   await waku.relay.send(wakuMessage);
 }
 

guides/relay-receive-send-messages.md

@@ -4,7 +4,12 @@ Waku
 Relay
 is
 a
-gossip protocol that enables you to send and receive messages.
+gossip
+protocol
+that
+enables
+you
+to send and receive messages.
 You can find Waku Relay's specifications on [Vac RFC](https://rfc.vac.dev/spec/11/).
 
 Before starting, you need to choose a _Content Topic_ for your dApp.
@@ -50,12 +55,15 @@ To monitor messages for your app, you need to register an observer on relay for
 
 ```js
 const processIncomingMessage = (wakuMessage) => {
-  console.log("Message Received", wakuMessage);
+  console.log(`Message Received: ${wakuMessage.payloadAsUtf8}`);
 };
 
-waku.relay.addObserver(processIncomingMessage, ["/relay-guide/1/chat/proto"]);
+waku.relay.addObserver(processIncomingMessage, ['/relay-guide/1/chat/proto']);
 ```
 
+`WakuMessage.payloadAsUtf8` is a nice helper to show UTF-8 encoding messages.
+However, you will probably need more structure messages, this is covered in [use protobuf section](#use-protobuf).
+
 # Send messages
 
 You are now ready to send messages.
@@ -67,20 +75,110 @@ When using a basic string payload, you can use the `WakuMessage.fromUtf8String`
 ```js
 import { WakuMessage } from 'js-waku';
 
-const wakuMessage = await WakuMessage.fromUtf8String(message, `/relay-guide/1/chat/proto`);
+const wakuMessage = await WakuMessage.fromUtf8String('Here is a message', `/relay-guide/1/chat/proto`);
 ```
 
 Then, use the `relay` module to send the message to our peers,
 the message will then be relayed to the rest of the network thanks to Waku Relay:
 
 ```js
-import { WakuMessage } from 'js-waku';
-
-const wakuMessage = await WakuMessage.fromUtf8String(message, `/relay-guide/1/chat/proto`);
-
 await waku.relay.send(wakuMessage);
 ```
 
+# Use protobuf
+
+Sending strings as messages in unlikely to cover your dApps needs.
+To include structured objects in Waku Messages,
+it is recommended to use [protobuf](https://developers.google.com/protocol-buffers/).
+
+First, let's define an object.
+For this guide, we will use a simple chat message that contains a timestamp and text:
+
+```js
+{
+  timestamp: Date;
+  text: string;
+}
+```
+
+To encode and decode protobuf, you can use the [protons](https://www.npmjs.com/package/protons) package.
+
+## Install protobuf library
+
+First, install it:
+
+```shell
+npm install protons
+```
+
+## Protobuf Definition
+
+Then define the simple chat message:
+
+```js
+import protons from 'protons';
+
+const proto = protons(`
+message SimpleChatMessage {
+  float timestamp = 1;
+  string text = 2;
+}
+`);
+```
+
+You can learn about protobuf definitions here:
+[Protocol Buffers Language Guide](https://developers.google.com/protocol-buffers/docs/proto).
+
+## Encode messages
+
+Instead of wrapping a string in a Waku Message, you need to encode the message in protobuf.
+The result is a byte array that can then be wrapped in a Waku Message.
+
+First, encode the message:
+
+```js
+const payload = proto.SimpleChatMessage.encode({
+  timestamp: Date.now(),
+  text: 'Here is a message'
+});
+```
+
+Then, wrap it in a Waku Message:
+
+```js
+const wakuMessage = await WakuMessage.fromBytes(payload, ContentTopic);
+```
+
+Now, you can send the message over Waku Relay the same way than before:
+
+```js
+await waku.relay.send(wakuMessage);
+```
+
+## Decode messages
+
+To decode the messages received over Waku Relay,
+you need to extract the protobuf payload and decode it using `protons`.
+
+```js
+const processIncomingMessage = (wakuMessage) => {
+  // No need to attempt to decode a message if the payload is absent
+  if (!wakuMessage.payload) return;
+
+  const { timestamp, text } = proto.SimpleChatMessage.decode(
+    wakuMessage.payload
+  );
+
+  console.log(`Message Received: ${text}, sent at ${timestamp.toString()}`);
+};
+```
+
+Same than before, you can pass add this function as an observer to Waku Relay to process incoming messages:
+
+```js
+waku.relay.addObserver(processIncomingMessage, ['/relay-guide/1/chat/proto']);
+```
+
 # Conclusion
 
 That is it! Now, you know how to send and receive messages over Waku using the Waku Relay protocol.
@@ -91,6 +189,14 @@ Here is the final code:
 
 ```js
 import { getStatusFleetNodes, Waku, WakuMessage } from 'js-waku';
+import protons from 'protons';
+
+const proto = protons(`
+message SimpleChatMessage {
+  float timestamp = 1;
+  string text = 2;
+}
+`);
 
 const wakuNode = await Waku.create();
 
@@ -98,11 +204,22 @@ const nodes = await getStatusFleetNodes();
 await Promise.all(nodes.map((addr) => waku.dial(addr)));
 
 const processIncomingMessage = (wakuMessage) => {
-  console.log(`Message Received: ${wakuMessage.payloadAsUtf8}`);
+  // No need to attempt to decode a message if the payload is absent
+  if (!wakuMessage.payload) return;
+
+  const { timestamp, text } = proto.SimpleChatMessage.decode(
+    wakuMessage.payload
+  );
+
+  console.log(`Message Received: ${text}, sent at ${timestamp.toString()}`);
 };
 
 waku.relay.addObserver(processIncomingMessage, ['/relay-guide/1/chat/proto']);
 
-const wakuMessage = await WakuMessage.fromUtf8String(message, `/relay-guide/1/chat/proto`);
+const payload = proto.SimpleChatMessage.encode({
+  timestamp: Date.now(),
+  text: 'Here is a message'
+});
+const wakuMessage = await WakuMessage.fromBytes(payload, ContentTopic);
 await waku.relay.send(wakuMessage);
 ```