From 36a01c3c300279e437b17f4d375ebd5fa757ddf3 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 21 Nov 2022 13:20:21 +0530 Subject: [PATCH 1/4] address comments --- packages/core/src/lib/waku_store/index.ts | 24 +++++++++++++++++------ packages/interfaces/src/index.ts | 8 +++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/core/src/lib/waku_store/index.ts b/packages/core/src/lib/waku_store/index.ts index 44e4f5087b..95ea807448 100644 --- a/packages/core/src/lib/waku_store/index.ts +++ b/packages/core/src/lib/waku_store/index.ts @@ -4,7 +4,13 @@ import type { PeerId } from "@libp2p/interface-peer-id"; import type { Peer, PeerStore } from "@libp2p/interface-peer-store"; import { sha256 } from "@noble/hashes/sha256"; import { concat, utf8ToBytes } from "@waku/byte-utils"; -import { DecodedMessage, Decoder, Index, Store } from "@waku/interfaces"; +import { + Cursor, + DecodedMessage, + Decoder, + Index, + Store, +} from "@waku/interfaces"; import debug from "debug"; import all from "it-all"; import * as lp from "it-length-prefixed"; @@ -84,8 +90,10 @@ export interface QueryOptions { timeFilter?: TimeFilter; /** * Cursor as an index to start a query from. + * The cursor index will be exclusive (i.e. the message at the cursor index will not be included in the result). + * If undefined, the query will start from the beginning or end of the history, depending on the page direction. */ - cursor?: Index; + cursor?: Cursor; } /** @@ -289,7 +297,7 @@ async function* paginate( protocol: string, queryOpts: Params, decoders: Map>, - cursor?: Index + cursor?: Cursor ): AsyncGenerator[]> { if ( queryOpts.contentTopics.toString() !== @@ -300,8 +308,10 @@ async function* paginate( ); } + let currentCursor = cursor; + while (true) { - queryOpts = Object.assign(queryOpts, { cursor }); + queryOpts = Object.assign(queryOpts, { currentCursor }); const stream = await connection.newStream(protocol); const historyRpcQuery = HistoryRPC.createQuery(queryOpts); @@ -362,8 +372,8 @@ async function* paginate( return Promise.resolve(undefined); }); - cursor = response.pagingInfo?.cursor; - if (typeof cursor === "undefined") { + const nextCursor = response.pagingInfo?.cursor; + if (typeof nextCursor === "undefined") { // If the server does not return cursor then there is an issue, // Need to abort, or we end up in an infinite loop log( @@ -372,6 +382,8 @@ async function* paginate( break; } + currentCursor = nextCursor; + const responsePageSize = response.pagingInfo?.pageSize; const queryPageSize = historyRpcQuery.query?.pagingInfo?.pageSize; if ( diff --git a/packages/interfaces/src/index.ts b/packages/interfaces/src/index.ts index 39aeeacf44..021aeaa114 100644 --- a/packages/interfaces/src/index.ts +++ b/packages/interfaces/src/index.ts @@ -60,6 +60,12 @@ export interface TimeFilter { endTime: Date; } +export type Cursor = { + digest?: Uint8Array; + senderTime?: bigint; + pubsubTopic?: string; +}; + export type StoreQueryOptions = { /** * The direction in which pages are retrieved: @@ -83,7 +89,7 @@ export type StoreQueryOptions = { /** * Cursor as an index to start a query from. */ - cursor?: Index; + cursor?: Cursor; } & ProtocolOptions; export interface Store extends PointToPointProtocol { From 9c8d7a5cd4808e788740bb68c6cfbb171761eecd Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 22 Nov 2022 22:47:04 +0530 Subject: [PATCH 2/4] chore: redeploy --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cff194b6f2..8114b413c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: node-version: ${{ env.NODE_JS }} - uses: bahmutov/npm-install@v1 - run: npm run build - - run: npm run test:node > debug.log + - run: npm run test:node env: DEBUG: "waku:nwaku*,waku:test*" @@ -131,7 +131,7 @@ jobs: - uses: bahmutov/npm-install@v1 - run: npm run build - - run: npm run test:node > debug.log + - run: npm run test:node env: DEBUG: "waku:nwaku*,waku:test*" @@ -185,7 +185,7 @@ jobs: ./build/wakunode2 --help - run: npm run build - - run: npm run test:node > debug.log + - run: npm run test:node env: DEBUG: "waku:nwaku*,waku:test*" From b8c5d27a3c053fa84a8ae376771b5518d4d7c122 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 22 Nov 2022 22:53:15 +0530 Subject: [PATCH 3/4] chore: redeploy --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8114b413c2..64aa99d1b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - run: npm run build - run: npm run test:node env: - DEBUG: "waku:nwaku*,waku:test*" + DEBUG: "" - name: Upload debug logs on failure uses: actions/upload-artifact@v3 From 131493076513a88a3fa0cb88f690f23dafc97ac7 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 22 Nov 2022 23:03:31 +0530 Subject: [PATCH 4/4] fix: store --- packages/core/src/lib/waku_store/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/core/src/lib/waku_store/index.ts b/packages/core/src/lib/waku_store/index.ts index 95ea807448..b4a5626fa9 100644 --- a/packages/core/src/lib/waku_store/index.ts +++ b/packages/core/src/lib/waku_store/index.ts @@ -309,9 +309,8 @@ async function* paginate( } let currentCursor = cursor; - while (true) { - queryOpts = Object.assign(queryOpts, { currentCursor }); + queryOpts.cursor = currentCursor; const stream = await connection.newStream(protocol); const historyRpcQuery = HistoryRPC.createQuery(queryOpts);