Merge pull request #308 from status-im/store-guide

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- **Breaking**: Renamed `WakuStore.QueryOptions`'s `direction` to `pageDirection` (and its type) as it only affects the page ordering,
+  not the ordering of messages with the page.
+
+### Fixed
+- Docs: Ensure that `WakuStore`'s `QueryOptions` documentation is available [online](https://status-im.github.io/js-waku/docs/).
+
 ## [0.13.1] - 2021-09-21
 
 ### Fixed

examples/store-reactjs-chat/src/App.js

@@ -44,15 +44,25 @@ function App() {
   React.useEffect(() => {
     if (wakuStatus !== 'Connected') return;
 
+    const processMessages = (retrievedMessages) => {
+      const messages = retrievedMessages.map(decodeMessage).filter(Boolean);
+
+      setMessages((currentMessages) => {
+        return currentMessages.concat(messages.reverse());
+      });
+    };
+
+    const startTime = new Date();
+    // 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec
+    startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000);
+
     waku.store
-      .queryHistory([ContentTopic])
+      .queryHistory([ContentTopic], {
+        callback: processMessages,
+        timeFilter: { startTime, endTime: new Date() },
+      })
       .catch((e) => {
         console.log('Failed to retrieve messages', e);
-      })
-      .then((retrievedMessages) => {
-        const messages = retrievedMessages.map(decodeMessage).filter(Boolean);
-
-        setMessages(messages);
       });
   }, [waku, wakuStatus]);
 
@@ -104,6 +114,7 @@ function formatDate(timestamp) {
     day: 'numeric',
     hour: 'numeric',
     minute: '2-digit',
+    second: '2-digit',
     hour12: false,
   });
 }

examples/web-chat/src/App.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useReducer, useState } from 'react';
 import './App.css';
-import { Direction, getBootstrapNodes, Waku, WakuMessage } from 'js-waku';
+import { PageDirection, getBootstrapNodes, Waku, WakuMessage } from 'js-waku';
 import handleCommand from './command';
 import Room from './Room';
 import { WakuContext } from './WakuContext';
@@ -64,7 +64,7 @@ async function retrieveStoreMessages(
   try {
     const res = await waku.store.queryHistory([ChatContentTopic], {
       pageSize: 5,
-      direction: Direction.FORWARD,
+      pageDirection: PageDirection.FORWARD,
       timeFilter: {
         startTime,
         endTime,

guides/reactjs-store.md

@@ -185,7 +185,41 @@ function decodeMessage(wakuMessage) {
 
 You now have all the building blocks to retrieve and decode messages for a store node.
 
-Finally, retrieve messages from a store node:
+Note that Waku Store queries are paginated.
+The API provided by `js-waku` automatically traverses all pages of the Waku Store response.
+By default, the most recent page is retrieved first but this can be changed with the `pageDirection` option.
+
+First, define a React state to save the messages:
+
+```js
+function App() {
+  const [messages, setMessages] = React.useState([]);
+  /// [..]
+}
+```
+
+Then, define `processMessages` to decode and then store messages in the React state.
+You will pass `processMessages` as a `callback` option to `WakuStore.queryHistory`.
+`processMessages` will be called each time a page is received from the Waku Store.
+
+```js
+const processMessages = (retrievedMessages) => {
+  const messages = retrievedMessages.map(decodeMessage).filter(Boolean);
+
+  setMessages((currentMessages) => {
+    return currentMessages.concat(messages.reverse());
+  });
+};
+```
+
+Finally, pass `processMessage` in `WakuStore.queryHistory` as the `callback` value:
+
+```js
+waku.store
+  .queryHistory([ContentTopic], { callback: processMessages });
+```
+
+All together, you should now have:
 
 ```js
 const ContentTopic = '/toy-chat/2/huilong/proto';
@@ -198,15 +232,18 @@ function App() {
   React.useEffect(() => {
     if (wakuStatus !== 'Connected') return;
 
+    const processMessages = (retrievedMessages) => {
+      const messages = retrievedMessages.map(decodeMessage).filter(Boolean);
+
+      setMessages((currentMessages) => {
+        return currentMessages.concat(messages.reverse());
+      });
+    };
+
     waku.store
-      .queryHistory([ContentTopic])
+      .queryHistory([ContentTopic], { callback: processMessages })
       .catch((e) => {
         console.log('Failed to retrieve messages', e);
-      })
-      .then((retrievedMessages) => {
-        const messages = retrievedMessages.map(decodeMessage).filter(Boolean);
-
-        setMessages(messages);
       });
   }, [waku, wakuStatus]);
 
@@ -229,4 +266,31 @@ Note that `WakuStore.queryHistory` select an available store node for you.
 However, it can only select a connected node, which is why the bootstrapping is necessary.
 It will throw an error if no store node is available.
 
+## Filter messages by send time
+
+By default, Waku Store nodes store messages for 30 days.
+Depending on your use case, you may not need to retrieve 30 days worth of messages.
+
+[Waku Message](https://rfc.vac.dev/spec/14/) defines an optional unencrypted `timestamp` field.
+The timestamp is set by the sender.
+By default, js-waku [sets the timestamp of outgoing message to the current time](https://github.com/status-im/js-waku/blob/a056227538f9409aa9134c7ef0df25f602dbea58/src/lib/waku_message/index.ts#L76).
+
+You can filter messages that include a timestamp within given bounds with the `timeFilter` option.
+
+Retrieve messages up to a week old:
+
+```js
+const startTime = new Date();
+// 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec
+startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000);
+
+waku.store
+  .queryHistory([ContentTopic], {
+    callback: processMessages,
+    timeFilter: { startTime, endTime: new Date() }
+  });
+```
+
+## End result
+
 You can see the complete code in the [Minimal ReactJS Waku Store App](/examples/store-reactjs-chat).

guides/store-retrieve-messages.md

@@ -140,23 +140,26 @@ const decodeWakuMessage = (wakuMessage) => {
 
 You now have all the building blocks to retrieve and decode messages for a store node.
 
-Retrieve messages from a store node:
+Store node responses are paginated.
+The `WakuStore.queryHistory` API automatically query all the pages in a sequential manner.
+To process messages as soon as they received (page by page), use the `callback` option:
 
 ```js
 const ContentTopic = '/store-guide/1/news/proto';
 
-waku.store
-  .queryHistory([ContentTopic])
-  .catch((e) => {
-    // Be sure to catch any potential error
-    console.log('Failed to retrieve messages', e);
-  })
-  .then((retrievedMessages) => {
-    const articles = retrievedMessages
-      .map(decodeWakuMessage) // Decode messages
-      .filter(Boolean); // Filter out undefined values
+const callback = (retrievedMessages) => {
+  const articles = retrievedMessages
+    .map(decodeWakuMessage) // Decode messages
+    .filter(Boolean); // Filter out undefined values
 
-    console.log(`${articles.length} articles have been retrieved`);
+  console.log(`${articles.length} articles have been retrieved`);
+};
+
+waku.store
+  .queryHistory([ContentTopic], { callback })
+  .catch((e) => {
+    // Catch any potential error
+    console.log('Failed to retrieve messages from store', e);
   });
 ```
 
@@ -164,3 +167,36 @@ Note that `WakuStore.queryHistory` select an available store node for you.
 However, it can only select a connected node, which is why the bootstrapping is necessary.
 It will throw an error if no store node is available.
 
+## Filter messages by send time
+
+By default, Waku Store nodes store messages for 30 days.
+Depending on your use case, you may not need to retrieve 30 days worth of messages.
+
+[Waku Message](https://rfc.vac.dev/spec/14/) defiles an optional unencrypted `timestamp` field.
+The timestamp is set by the sender.
+By default, js-waku [sets the timestamp of outgoing message to the current time](https://github.com/status-im/js-waku/blob/a056227538f9409aa9134c7ef0df25f602dbea58/src/lib/waku_message/index.ts#L76).
+
+You can filter messages that include a timestamp within given bounds with the `timeFilter` option.
+
+Retrieve messages up to a week old:
+
+```js
+// [..] `ContentTopic` and `callback` definitions
+
+const startTime = new Date();
+// 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec
+startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000);
+
+waku.store
+  .queryHistory([ContentTopic], {
+    callback,
+    timeFilter: { startTime, endTime: new Date() }
+  })
+  .catch((e) => {
+    console.log('Failed to retrieve messages from store', e);
+  });
+```
+
+## End result
+
+You can see a similar example implemented in ReactJS in the [Minimal ReactJS Waku Store App](/examples/store-reactjs-chat).

src/index.ts

@@ -20,6 +20,6 @@ export {
 
 export { WakuRelay, RelayCodecs } from './lib/waku_relay';
 
-export { Direction, WakuStore, StoreCodec } from './lib/waku_store';
+export { PageDirection, WakuStore, StoreCodec } from './lib/waku_store';
 
 export * as proto from './proto';

src/lib/waku.ts

@@ -328,7 +328,6 @@ export class Waku {
             peers.push(peer)
           );
         });
-        dbg('peers for ', desiredProtocolVersions, peers);
 
         if (peers.length > 0) {
           return Promise.resolve();

src/lib/waku_store/history_rpc.ts

@@ -3,7 +3,7 @@ import { v4 as uuid } from 'uuid';
 
 import * as proto from '../../proto/waku/v2/store';
 
-export enum Direction {
+export enum PageDirection {
   BACKWARD = 'backward',
   FORWARD = 'forward',
 }
@@ -11,7 +11,7 @@ export enum Direction {
 export interface Params {
   contentTopics: string[];
   pubSubTopic: string;
-  direction: Direction;
+  pageDirection: PageDirection;
   pageSize: number;
   startTime?: number;
   endTime?: number;
@@ -25,7 +25,7 @@ export class HistoryRPC {
    * Create History Query.
    */
   static createQuery(params: Params): HistoryRPC {
-    const direction = directionToProto(params.direction);
+    const direction = directionToProto(params.pageDirection);
     const pagingInfo = {
       pageSize: params.pageSize,
       cursor: params.cursor,
@@ -67,11 +67,13 @@ export class HistoryRPC {
   }
 }
 
-function directionToProto(direction: Direction): proto.PagingInfo_Direction {
-  switch (direction) {
-    case Direction.BACKWARD:
+function directionToProto(
+  pageDirection: PageDirection
+): proto.PagingInfo_Direction {
+  switch (pageDirection) {
+    case PageDirection.BACKWARD:
       return proto.PagingInfo_Direction.DIRECTION_BACKWARD_UNSPECIFIED;
-    case Direction.FORWARD:
+    case PageDirection.FORWARD:
       return proto.PagingInfo_Direction.DIRECTION_FORWARD;
     default:
       return proto.PagingInfo_Direction.DIRECTION_BACKWARD_UNSPECIFIED;

src/lib/waku_store/index.node.spec.ts

@@ -19,7 +19,7 @@ import {
   getPublicKey,
 } from '../waku_message/version_1';
 
-import { Direction } from './history_rpc';
+import { PageDirection } from './history_rpc';
 
 const dbg = debug('waku:test:store');
 
@@ -94,7 +94,7 @@ describe('Waku Store', () => {
     });
 
     const messages = await waku.store.queryHistory([], {
-      direction: Direction.FORWARD,
+      pageDirection: PageDirection.FORWARD,
     });
 
     expect(messages?.length).eq(15);

src/lib/waku_store/index.ts

@@ -12,13 +12,15 @@ import { hexToBuf } from '../utils';
 import { DefaultPubSubTopic } from '../waku';
 import { WakuMessage } from '../waku_message';
 
-import { Direction, HistoryRPC } from './history_rpc';
+import { HistoryRPC, PageDirection } from './history_rpc';
 
 const dbg = debug('waku:store');
 
 export const StoreCodec = '/vac/waku/store/2.0.0-beta3';
 
-export { Direction };
+export const DefaultPageSize = 10;
+
+export { PageDirection };
 
 export interface CreateOptions {
   /**
@@ -38,12 +40,50 @@ export interface TimeFilter {
 }
 
 export interface QueryOptions {
+  /**
+   * The peer to query. If undefined, a pseudo-random peer is selected from the connected Waku Store peers.
+   */
   peerId?: PeerId;
+  /**
+   * The pubsub topic to pass to the query.
+   * See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/).
+   */
   pubSubTopic?: string;
-  direction?: Direction;
+  /**
+   * The direction in which pages are retrieved:
+   * - [[Direction.BACKWARD]]: Most recent page first.
+   * - [[Direction.FORWARD]]: Oldest page first.
+   *
+   * Note: This does not affect the ordering of messages with the page
+   * (oldest message is always first).
+   *
+   * @default [[Direction.BACKWARD]]
+   */
+  pageDirection?: PageDirection;
+  /**
+   * The number of message per page.
+   *
+   * @default [[DefaultPageSize]]
+   */
   pageSize?: number;
+  /**
+   * Retrieve messages with a timestamp within the provided values.
+   */
   timeFilter?: TimeFilter;
+  /**
+   * Callback called on pages of stored messages as they are retrieved.
+   * Allows for a faster access to the results as it is called as soon as a page
+   * is received.
+   * Traversal of the pages is done automatically so this function will invoked
+   * for each retrieved page.
+   */
   callback?: (messages: WakuMessage[]) => void;
+  /**
+   * Keys that will be used to decrypt messages.
+   *
+   * It can be Asymmetric Private Keys and Symmetric Keys in the same array,
+   * all keys will be tried with both methods.
+   */
   decryptionKeys?: Array<Uint8Array | string>;
 }
 
@@ -65,20 +105,13 @@ export class WakuStore {
   }
 
   /**
-   * Query given peer using Waku Store.
+   * Do a History Query to a Waku Store.
    *
    * @param contentTopics The content topics to pass to the query, leave empty to
    * retrieve all messages.
-   * @param options
-   * @param options.peerId The peer to query.Options
-   * @param options.timeFilter Query messages with a timestamp within the provided values.
-   * @param options.pubSubTopic The pubsub topic to pass to the query. Defaults
-   * to the value set at creation. See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/).
-   * @param options.callback Callback called on page of stored messages as they are retrieved
-   * @param options.decryptionKeys Keys that will be used to decrypt messages.
-   * It can be Asymmetric Private Keys and Symmetric Keys in the same array, all keys will be tried with both
-   * methods.
-   * @throws If not able to reach the peer to query or error when processing the reply.
+   *
+   * @throws If not able to reach a Waku Store peer to query
+   * or if an error is encountered when processing the reply.
    */
   async queryHistory(
     contentTopics: string[],
@@ -93,8 +126,8 @@ export class WakuStore {
     const opts = Object.assign(
       {
         pubSubTopic: this.pubSubTopic,
-        direction: Direction.BACKWARD,
-        pageSize: 10,
+        pageDirection: PageDirection.BACKWARD,
+        pageSize: DefaultPageSize,
       },
       options,
       {
@@ -133,6 +166,8 @@ export class WakuStore {
       const { stream } = await connection.newStream(StoreCodec);
       const queryOpts = Object.assign(opts, { cursor });
       const historyRpcQuery = HistoryRPC.createQuery(queryOpts);
+      dbg('Querying store peer', connection.remoteAddr.toString());
+
       const res = await pipe(
         [historyRpcQuery.encode()],
         lp.encode(),