diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e107ce43..ea8e2166df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Breaking**: Renamed `WakuStore.QueryOptions`'s `direction` to `pageDirection` (and its type) as it only affects the page ordering, + not the ordering of messages with the page. + +### Fixed +- Docs: Ensure that `WakuStore`'s `QueryOptions` documentation is available [online](https://status-im.github.io/js-waku/docs/). + ## [0.13.1] - 2021-09-21 ### Fixed diff --git a/examples/store-reactjs-chat/src/App.js b/examples/store-reactjs-chat/src/App.js index 8e7566e05e..2ad02e624f 100644 --- a/examples/store-reactjs-chat/src/App.js +++ b/examples/store-reactjs-chat/src/App.js @@ -44,15 +44,25 @@ function App() { React.useEffect(() => { if (wakuStatus !== 'Connected') return; + const processMessages = (retrievedMessages) => { + const messages = retrievedMessages.map(decodeMessage).filter(Boolean); + + setMessages((currentMessages) => { + return currentMessages.concat(messages.reverse()); + }); + }; + + const startTime = new Date(); + // 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec + startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000); + waku.store - .queryHistory([ContentTopic]) + .queryHistory([ContentTopic], { + callback: processMessages, + timeFilter: { startTime, endTime: new Date() }, + }) .catch((e) => { console.log('Failed to retrieve messages', e); - }) - .then((retrievedMessages) => { - const messages = retrievedMessages.map(decodeMessage).filter(Boolean); - - setMessages(messages); }); }, [waku, wakuStatus]); @@ -104,6 +114,7 @@ function formatDate(timestamp) { day: 'numeric', hour: 'numeric', minute: '2-digit', + second: '2-digit', hour12: false, }); } diff --git a/examples/web-chat/src/App.tsx b/examples/web-chat/src/App.tsx index bebbe23cc5..bc4eaa9e8d 100644 --- a/examples/web-chat/src/App.tsx +++ b/examples/web-chat/src/App.tsx @@ -1,6 +1,6 @@ import { useEffect, useReducer, useState } from 'react'; import './App.css'; -import { Direction, getBootstrapNodes, Waku, WakuMessage } from 'js-waku'; +import { PageDirection, getBootstrapNodes, Waku, WakuMessage } from 'js-waku'; import handleCommand from './command'; import Room from './Room'; import { WakuContext } from './WakuContext'; @@ -64,7 +64,7 @@ async function retrieveStoreMessages( try { const res = await waku.store.queryHistory([ChatContentTopic], { pageSize: 5, - direction: Direction.FORWARD, + pageDirection: PageDirection.FORWARD, timeFilter: { startTime, endTime, diff --git a/guides/reactjs-store.md b/guides/reactjs-store.md index edccb59ec4..7bd2d462e1 100644 --- a/guides/reactjs-store.md +++ b/guides/reactjs-store.md @@ -185,7 +185,41 @@ function decodeMessage(wakuMessage) { You now have all the building blocks to retrieve and decode messages for a store node. -Finally, retrieve messages from a store node: +Note that Waku Store queries are paginated. +The API provided by `js-waku` automatically traverses all pages of the Waku Store response. +By default, the most recent page is retrieved first but this can be changed with the `pageDirection` option. + +First, define a React state to save the messages: + +```js +function App() { + const [messages, setMessages] = React.useState([]); + /// [..] +} +``` + +Then, define `processMessages` to decode and then store messages in the React state. +You will pass `processMessages` as a `callback` option to `WakuStore.queryHistory`. +`processMessages` will be called each time a page is received from the Waku Store. + +```js +const processMessages = (retrievedMessages) => { + const messages = retrievedMessages.map(decodeMessage).filter(Boolean); + + setMessages((currentMessages) => { + return currentMessages.concat(messages.reverse()); + }); +}; +``` + +Finally, pass `processMessage` in `WakuStore.queryHistory` as the `callback` value: + +```js +waku.store + .queryHistory([ContentTopic], { callback: processMessages }); +``` + +All together, you should now have: ```js const ContentTopic = '/toy-chat/2/huilong/proto'; @@ -198,15 +232,18 @@ function App() { React.useEffect(() => { if (wakuStatus !== 'Connected') return; + const processMessages = (retrievedMessages) => { + const messages = retrievedMessages.map(decodeMessage).filter(Boolean); + + setMessages((currentMessages) => { + return currentMessages.concat(messages.reverse()); + }); + }; + waku.store - .queryHistory([ContentTopic]) + .queryHistory([ContentTopic], { callback: processMessages }) .catch((e) => { console.log('Failed to retrieve messages', e); - }) - .then((retrievedMessages) => { - const messages = retrievedMessages.map(decodeMessage).filter(Boolean); - - setMessages(messages); }); }, [waku, wakuStatus]); @@ -229,4 +266,31 @@ Note that `WakuStore.queryHistory` select an available store node for you. However, it can only select a connected node, which is why the bootstrapping is necessary. It will throw an error if no store node is available. +## Filter messages by send time + +By default, Waku Store nodes store messages for 30 days. +Depending on your use case, you may not need to retrieve 30 days worth of messages. + +[Waku Message](https://rfc.vac.dev/spec/14/) defines an optional unencrypted `timestamp` field. +The timestamp is set by the sender. +By default, js-waku [sets the timestamp of outgoing message to the current time](https://github.com/status-im/js-waku/blob/a056227538f9409aa9134c7ef0df25f602dbea58/src/lib/waku_message/index.ts#L76). + +You can filter messages that include a timestamp within given bounds with the `timeFilter` option. + +Retrieve messages up to a week old: + +```js +const startTime = new Date(); +// 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec +startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000); + +waku.store + .queryHistory([ContentTopic], { + callback: processMessages, + timeFilter: { startTime, endTime: new Date() } + }); +``` + +## End result + You can see the complete code in the [Minimal ReactJS Waku Store App](/examples/store-reactjs-chat). diff --git a/guides/store-retrieve-messages.md b/guides/store-retrieve-messages.md index dbab771cbd..e37c4b856e 100644 --- a/guides/store-retrieve-messages.md +++ b/guides/store-retrieve-messages.md @@ -140,23 +140,26 @@ const decodeWakuMessage = (wakuMessage) => { You now have all the building blocks to retrieve and decode messages for a store node. -Retrieve messages from a store node: +Store node responses are paginated. +The `WakuStore.queryHistory` API automatically query all the pages in a sequential manner. +To process messages as soon as they received (page by page), use the `callback` option: ```js const ContentTopic = '/store-guide/1/news/proto'; -waku.store - .queryHistory([ContentTopic]) - .catch((e) => { - // Be sure to catch any potential error - console.log('Failed to retrieve messages', e); - }) - .then((retrievedMessages) => { - const articles = retrievedMessages - .map(decodeWakuMessage) // Decode messages - .filter(Boolean); // Filter out undefined values +const callback = (retrievedMessages) => { + const articles = retrievedMessages + .map(decodeWakuMessage) // Decode messages + .filter(Boolean); // Filter out undefined values - console.log(`${articles.length} articles have been retrieved`); + console.log(`${articles.length} articles have been retrieved`); +}; + +waku.store + .queryHistory([ContentTopic], { callback }) + .catch((e) => { + // Catch any potential error + console.log('Failed to retrieve messages from store', e); }); ``` @@ -164,3 +167,36 @@ Note that `WakuStore.queryHistory` select an available store node for you. However, it can only select a connected node, which is why the bootstrapping is necessary. It will throw an error if no store node is available. +## Filter messages by send time + +By default, Waku Store nodes store messages for 30 days. +Depending on your use case, you may not need to retrieve 30 days worth of messages. + +[Waku Message](https://rfc.vac.dev/spec/14/) defiles an optional unencrypted `timestamp` field. +The timestamp is set by the sender. +By default, js-waku [sets the timestamp of outgoing message to the current time](https://github.com/status-im/js-waku/blob/a056227538f9409aa9134c7ef0df25f602dbea58/src/lib/waku_message/index.ts#L76). + +You can filter messages that include a timestamp within given bounds with the `timeFilter` option. + +Retrieve messages up to a week old: + +```js +// [..] `ContentTopic` and `callback` definitions + +const startTime = new Date(); +// 7 days/week, 24 hours/day, 60min/hour, 60secs/min, 100ms/sec +startTime.setTime(startTime.getTime() - 7 * 24 * 60 * 60 * 1000); + +waku.store + .queryHistory([ContentTopic], { + callback, + timeFilter: { startTime, endTime: new Date() } + }) + .catch((e) => { + console.log('Failed to retrieve messages from store', e); + }); +``` + +## End result + +You can see a similar example implemented in ReactJS in the [Minimal ReactJS Waku Store App](/examples/store-reactjs-chat). diff --git a/src/index.ts b/src/index.ts index 69c667b160..324459c865 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,6 @@ export { export { WakuRelay, RelayCodecs } from './lib/waku_relay'; -export { Direction, WakuStore, StoreCodec } from './lib/waku_store'; +export { PageDirection, WakuStore, StoreCodec } from './lib/waku_store'; export * as proto from './proto'; diff --git a/src/lib/waku.ts b/src/lib/waku.ts index 286cd3487e..6900ab6e07 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -328,7 +328,6 @@ export class Waku { peers.push(peer) ); }); - dbg('peers for ', desiredProtocolVersions, peers); if (peers.length > 0) { return Promise.resolve(); diff --git a/src/lib/waku_store/history_rpc.ts b/src/lib/waku_store/history_rpc.ts index cdf328fb57..2e2194c981 100644 --- a/src/lib/waku_store/history_rpc.ts +++ b/src/lib/waku_store/history_rpc.ts @@ -3,7 +3,7 @@ import { v4 as uuid } from 'uuid'; import * as proto from '../../proto/waku/v2/store'; -export enum Direction { +export enum PageDirection { BACKWARD = 'backward', FORWARD = 'forward', } @@ -11,7 +11,7 @@ export enum Direction { export interface Params { contentTopics: string[]; pubSubTopic: string; - direction: Direction; + pageDirection: PageDirection; pageSize: number; startTime?: number; endTime?: number; @@ -25,7 +25,7 @@ export class HistoryRPC { * Create History Query. */ static createQuery(params: Params): HistoryRPC { - const direction = directionToProto(params.direction); + const direction = directionToProto(params.pageDirection); const pagingInfo = { pageSize: params.pageSize, cursor: params.cursor, @@ -67,11 +67,13 @@ export class HistoryRPC { } } -function directionToProto(direction: Direction): proto.PagingInfo_Direction { - switch (direction) { - case Direction.BACKWARD: +function directionToProto( + pageDirection: PageDirection +): proto.PagingInfo_Direction { + switch (pageDirection) { + case PageDirection.BACKWARD: return proto.PagingInfo_Direction.DIRECTION_BACKWARD_UNSPECIFIED; - case Direction.FORWARD: + case PageDirection.FORWARD: return proto.PagingInfo_Direction.DIRECTION_FORWARD; default: return proto.PagingInfo_Direction.DIRECTION_BACKWARD_UNSPECIFIED; diff --git a/src/lib/waku_store/index.node.spec.ts b/src/lib/waku_store/index.node.spec.ts index b11daf570f..6fb739a0ac 100644 --- a/src/lib/waku_store/index.node.spec.ts +++ b/src/lib/waku_store/index.node.spec.ts @@ -19,7 +19,7 @@ import { getPublicKey, } from '../waku_message/version_1'; -import { Direction } from './history_rpc'; +import { PageDirection } from './history_rpc'; const dbg = debug('waku:test:store'); @@ -94,7 +94,7 @@ describe('Waku Store', () => { }); const messages = await waku.store.queryHistory([], { - direction: Direction.FORWARD, + pageDirection: PageDirection.FORWARD, }); expect(messages?.length).eq(15); diff --git a/src/lib/waku_store/index.ts b/src/lib/waku_store/index.ts index 1b2286186f..613be4d4c1 100644 --- a/src/lib/waku_store/index.ts +++ b/src/lib/waku_store/index.ts @@ -12,13 +12,15 @@ import { hexToBuf } from '../utils'; import { DefaultPubSubTopic } from '../waku'; import { WakuMessage } from '../waku_message'; -import { Direction, HistoryRPC } from './history_rpc'; +import { HistoryRPC, PageDirection } from './history_rpc'; const dbg = debug('waku:store'); export const StoreCodec = '/vac/waku/store/2.0.0-beta3'; -export { Direction }; +export const DefaultPageSize = 10; + +export { PageDirection }; export interface CreateOptions { /** @@ -38,12 +40,50 @@ export interface TimeFilter { } export interface QueryOptions { + /** + * The peer to query. If undefined, a pseudo-random peer is selected from the connected Waku Store peers. + */ peerId?: PeerId; + /** + * The pubsub topic to pass to the query. + * See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/). + */ pubSubTopic?: string; - direction?: Direction; + /** + * The direction in which pages are retrieved: + * - [[Direction.BACKWARD]]: Most recent page first. + * - [[Direction.FORWARD]]: Oldest page first. + * + * Note: This does not affect the ordering of messages with the page + * (oldest message is always first). + * + * @default [[Direction.BACKWARD]] + */ + pageDirection?: PageDirection; + /** + * The number of message per page. + * + * @default [[DefaultPageSize]] + */ pageSize?: number; + /** + * Retrieve messages with a timestamp within the provided values. + */ timeFilter?: TimeFilter; + /** + * Callback called on pages of stored messages as they are retrieved. + * Allows for a faster access to the results as it is called as soon as a page + * is received. + * Traversal of the pages is done automatically so this function will invoked + * for each retrieved page. + */ callback?: (messages: WakuMessage[]) => void; + /** + * Keys that will be used to decrypt messages. + * + * It can be Asymmetric Private Keys and Symmetric Keys in the same array, + * all keys will be tried with both methods. + */ decryptionKeys?: Array; } @@ -65,20 +105,13 @@ export class WakuStore { } /** - * Query given peer using Waku Store. + * Do a History Query to a Waku Store. * * @param contentTopics The content topics to pass to the query, leave empty to * retrieve all messages. - * @param options - * @param options.peerId The peer to query.Options - * @param options.timeFilter Query messages with a timestamp within the provided values. - * @param options.pubSubTopic The pubsub topic to pass to the query. Defaults - * to the value set at creation. See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/). - * @param options.callback Callback called on page of stored messages as they are retrieved - * @param options.decryptionKeys Keys that will be used to decrypt messages. - * It can be Asymmetric Private Keys and Symmetric Keys in the same array, all keys will be tried with both - * methods. - * @throws If not able to reach the peer to query or error when processing the reply. + * + * @throws If not able to reach a Waku Store peer to query + * or if an error is encountered when processing the reply. */ async queryHistory( contentTopics: string[], @@ -93,8 +126,8 @@ export class WakuStore { const opts = Object.assign( { pubSubTopic: this.pubSubTopic, - direction: Direction.BACKWARD, - pageSize: 10, + pageDirection: PageDirection.BACKWARD, + pageSize: DefaultPageSize, }, options, { @@ -133,6 +166,8 @@ export class WakuStore { const { stream } = await connection.newStream(StoreCodec); const queryOpts = Object.assign(opts, { cursor }); const historyRpcQuery = HistoryRPC.createQuery(queryOpts); + dbg('Querying store peer', connection.remoteAddr.toString()); + const res = await pipe( [historyRpcQuery.encode()], lp.encode(),