From f307e9b6c6ebac3bae35da1b1acb44cb9b6a0a4f Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 29 Sep 2023 14:03:43 +0300 Subject: [PATCH 01/15] intermediat commit --- packages/tests/src/node/node.ts | 4 +- packages/tests/src/teardown.ts | 35 +- packages/tests/tests/filter/ping.node.spec.ts | 3 +- packages/tests/tests/filter/push.node.spec.ts | 3 +- .../tests/tests/filter/subscribe.node.spec.ts | 3 +- .../tests/filter/unsubscribe.node.spec.ts | 3 +- packages/tests/tests/filter/utils.ts | 9 +- .../light-push/custom_pubsub.node.spec.ts | 3 +- packages/tests/tests/light-push/index.spec.ts | 3 +- packages/tests/tests/light-push/utils.ts | 13 +- .../tests/tests/store/multiple_pubsub.spec.ts | 87 +++ packages/tests/tests/store/store.node.spec.ts | 529 ++++++++++++++++++ packages/tests/tests/store/utils.ts | 60 ++ 13 files changed, 725 insertions(+), 30 deletions(-) create mode 100644 packages/tests/tests/store/multiple_pubsub.spec.ts create mode 100644 packages/tests/tests/store/store.node.spec.ts create mode 100644 packages/tests/tests/store/utils.ts diff --git a/packages/tests/src/node/node.ts b/packages/tests/src/node/node.ts index 5cd8c85d14..47345db09a 100644 --- a/packages/tests/src/node/node.ts +++ b/packages/tests/src/node/node.ts @@ -168,8 +168,8 @@ export class NimGoNode { async startWithRetries( args: Args, options: { - retries: number; - } + retries?: number; + } = { retries: 3 } ): Promise { await pRetry( async () => { diff --git a/packages/tests/src/teardown.ts b/packages/tests/src/teardown.ts index caecec6e44..c4605d9c43 100644 --- a/packages/tests/src/teardown.ts +++ b/packages/tests/src/teardown.ts @@ -1,23 +1,46 @@ import { LightNode } from "@waku/interfaces"; import debug from "debug"; +import pRetry from "p-retry"; import { NimGoNode } from "./index.js"; const log = debug("waku:test"); -export function tearDownNodes( +export async function tearDownNodes( nwakuNodes: NimGoNode[], wakuNodes: LightNode[] -): void { - nwakuNodes.forEach((nwaku) => { +): Promise { + const stopNwakuNodes = nwakuNodes.map(async (nwaku) => { if (nwaku) { - nwaku.stop().catch((e) => log("Nwaku failed to stop", e)); + await pRetry( + async () => { + try { + await nwaku.stop(); + } catch (error) { + log("Nwaku failed to stop:", error); + throw error; + } + }, + { retries: 3 } + ); } }); - wakuNodes.forEach((waku) => { + const stopWakuNodes = wakuNodes.map(async (waku) => { if (waku) { - waku.stop().catch((e) => log("Waku failed to stop", e)); + await pRetry( + async () => { + try { + await waku.stop(); + } catch (error) { + log("Waku failed to stop:", error); + throw error; + } + }, + { retries: 3 } + ); } }); + + await Promise.all([...stopNwakuNodes, ...stopWakuNodes]); } diff --git a/packages/tests/tests/filter/ping.node.spec.ts b/packages/tests/tests/filter/ping.node.spec.ts index b35da7281b..c911e819d1 100644 --- a/packages/tests/tests/filter/ping.node.spec.ts +++ b/packages/tests/tests/filter/ping.node.spec.ts @@ -28,7 +28,8 @@ describe("Waku Filter V2: Ping", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); }); it("Ping on subscribed peer", async function () { diff --git a/packages/tests/tests/filter/push.node.spec.ts b/packages/tests/tests/filter/push.node.spec.ts index b695193d34..3a37fdac2c 100644 --- a/packages/tests/tests/filter/push.node.spec.ts +++ b/packages/tests/tests/filter/push.node.spec.ts @@ -37,7 +37,8 @@ describe("Waku Filter V2: FilterPush", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); }); TEST_STRING.forEach((testItem) => { diff --git a/packages/tests/tests/filter/subscribe.node.spec.ts b/packages/tests/tests/filter/subscribe.node.spec.ts index 0ded20d724..93bf672cab 100644 --- a/packages/tests/tests/filter/subscribe.node.spec.ts +++ b/packages/tests/tests/filter/subscribe.node.spec.ts @@ -48,7 +48,8 @@ describe("Waku Filter V2: Subscribe", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku, nwaku2], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku, nwaku2], [waku]); }); it("Subscribe and receive messages via lightPush", async function () { diff --git a/packages/tests/tests/filter/unsubscribe.node.spec.ts b/packages/tests/tests/filter/unsubscribe.node.spec.ts index b1591c2911..bf9a99b1a1 100644 --- a/packages/tests/tests/filter/unsubscribe.node.spec.ts +++ b/packages/tests/tests/filter/unsubscribe.node.spec.ts @@ -34,7 +34,8 @@ describe("Waku Filter V2: Unsubscribe", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); }); it("Unsubscribe 1 topic - node subscribed to 1 topic", async function () { diff --git a/packages/tests/tests/filter/utils.ts b/packages/tests/tests/filter/utils.ts index 988bf49dbb..1842af001b 100644 --- a/packages/tests/tests/filter/utils.ts +++ b/packages/tests/tests/filter/utils.ts @@ -67,14 +67,7 @@ export async function runNodes( currentTest: Context ): Promise<[NimGoNode, LightNode]> { const nwaku = new NimGoNode(makeLogFileName(currentTest)); - await nwaku.startWithRetries( - { - filter: true, - lightpush: true, - relay: true - }, - { retries: 3 } - ); + await nwaku.startWithRetries({ filter: true, lightpush: true, relay: true }); let waku: LightNode | undefined; try { diff --git a/packages/tests/tests/light-push/custom_pubsub.node.spec.ts b/packages/tests/tests/light-push/custom_pubsub.node.spec.ts index fa8247a786..d1aa4c6921 100644 --- a/packages/tests/tests/light-push/custom_pubsub.node.spec.ts +++ b/packages/tests/tests/light-push/custom_pubsub.node.spec.ts @@ -26,7 +26,8 @@ describe("Waku Light Push [node only] - custom pubsub topic", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); }); it("Push message", async function () { diff --git a/packages/tests/tests/light-push/index.spec.ts b/packages/tests/tests/light-push/index.spec.ts index db2e29680b..c62797c93e 100644 --- a/packages/tests/tests/light-push/index.spec.ts +++ b/packages/tests/tests/light-push/index.spec.ts @@ -39,7 +39,8 @@ describe("Waku Light Push [node only]", function () { }); this.afterEach(async function () { - tearDownNodes([nwaku], [waku]); + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); }); TEST_STRING.forEach((testItem) => { diff --git a/packages/tests/tests/light-push/utils.ts b/packages/tests/tests/light-push/utils.ts index 9e2d4b03b7..e669d10938 100644 --- a/packages/tests/tests/light-push/utils.ts +++ b/packages/tests/tests/light-push/utils.ts @@ -18,14 +18,11 @@ export async function runNodes( ): Promise<[NimGoNode, LightNode]> { const nwakuOptional = pubSubTopic ? { topic: pubSubTopic } : {}; const nwaku = new NimGoNode(makeLogFileName(context)); - await nwaku.startWithRetries( - { - lightpush: true, - relay: true, - ...nwakuOptional - }, - { retries: 3 } - ); + await nwaku.startWithRetries({ + lightpush: true, + relay: true, + ...nwakuOptional + }); let waku: LightNode | undefined; try { diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts new file mode 100644 index 0000000000..05367aa46a --- /dev/null +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -0,0 +1,87 @@ +import { createDecoder, waitForRemotePeer } from "@waku/core"; +import type { IMessage, LightNode } from "@waku/interfaces"; +import { Protocols } from "@waku/interfaces"; +import { createLightNode } from "@waku/sdk"; +import { expect } from "chai"; + +import { + makeLogFileName, + NimGoNode, + NOISE_KEY_1, + tearDownNodes +} from "../../src/index.js"; + +const customPubSubTopic = "/waku/2/custom-dapp/proto"; +const TestContentTopic = "/test/1/waku-store/utf8"; +const CustomPubSubTestDecoder = createDecoder( + TestContentTopic, + customPubSubTopic +); + +describe("Waku Store, custom pubsub topic", () => { + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ + store: true, + topic: customPubSubTopic, + relay: true + }); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + it("Generator, custom pubsub topic", async function () { + this.timeout(15000); + + const totalMsgs = 20; + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }), + customPubSubTopic + ) + ).to.be.true; + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1, + pubSubTopics: [customPubSubTopic] + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const messages: IMessage[] = []; + let promises: Promise[] = []; + for await (const msgPromises of waku.store.queryGenerator([ + CustomPubSubTestDecoder + ])) { + const _promises = msgPromises.map(async (promise) => { + const msg = await promise; + if (msg) { + messages.push(msg); + expect(msg.pubSubTopic).to.eq(customPubSubTopic); + } + }); + + promises = promises.concat(_promises); + } + await Promise.all(promises); + + expect(messages?.length).eq(totalMsgs); + const result = messages?.findIndex((msg) => { + return msg.payload![0]! === 0; + }); + expect(result).to.not.eq(-1); + }); +}); diff --git a/packages/tests/tests/store/store.node.spec.ts b/packages/tests/tests/store/store.node.spec.ts new file mode 100644 index 0000000000..4efd1389b9 --- /dev/null +++ b/packages/tests/tests/store/store.node.spec.ts @@ -0,0 +1,529 @@ +import { + createCursor, + createDecoder, + createEncoder, + DecodedMessage, + DefaultPubSubTopic, + PageDirection, + waitForRemotePeer +} from "@waku/core"; +import type { IMessage, LightNode } from "@waku/interfaces"; +import { Protocols } from "@waku/interfaces"; +import { + createDecoder as createEciesDecoder, + createEncoder as createEciesEncoder, + generatePrivateKey, + getPublicKey +} from "@waku/message-encryption/ecies"; +import { + createDecoder as createSymDecoder, + createEncoder as createSymEncoder, + generateSymmetricKey +} from "@waku/message-encryption/symmetric"; +import { createLightNode } from "@waku/sdk"; +import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; +import { expect } from "chai"; +import debug from "debug"; + +import { + delay, + makeLogFileName, + NimGoNode, + NOISE_KEY_1, + NOISE_KEY_2, + tearDownNodes +} from "../../src/index.js"; + +import { + processMessages, + sendMessages, + startAndConnectLightNode +} from "./utils.js"; + +const log = debug("waku:test:store"); + +const TestContentTopic = "/test/1/waku-store/utf8"; +const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); +const TestDecoder = createDecoder(TestContentTopic); + +describe.only("Waku Store", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + it("Generator", async function () { + const totalMsgs = 20; + + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + const messages = await processMessages( + waku, + [TestDecoder], + DefaultPubSubTopic + ); + + expect(messages?.length).eq(totalMsgs); + const result = messages?.findIndex((msg) => { + return msg.payload[0]! === 0; + }); + expect(result).to.not.eq(-1); + }); + + it.only("Generator, no message returned", async function () { + waku = await startAndConnectLightNode(nwaku); + const messages = await processMessages( + waku, + [TestDecoder], + DefaultPubSubTopic + ); + + expect(messages?.length).eq(0); + }); + + it("Passing a cursor", async function () { + this.timeout(4_000); + const totalMsgs = 20; + + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: utf8ToBytes(`Message ${i}`), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const query = waku.store.queryGenerator([TestDecoder]); + + // messages in reversed order (first message at last index) + const messages: DecodedMessage[] = []; + for await (const page of query) { + for await (const msg of page.reverse()) { + messages.push(msg as DecodedMessage); + } + } + + // index 2 would mean the third last message sent + const cursorIndex = 2; + + // create cursor to extract messages after the 3rd index + const cursor = await createCursor(messages[cursorIndex]); + + const messagesAfterCursor: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + messagesAfterCursor.push(msg as DecodedMessage); + } + } + + const testMessage = messagesAfterCursor[0]; + + expect(messages.length).be.eq(totalMsgs); + + expect(bytesToUtf8(testMessage.payload)).to.be.eq( + bytesToUtf8(messages[cursorIndex + 1].payload) + ); + }); + + it("Callback on promise", async function () { + this.timeout(15_000); + + const totalMsgs = 15; + + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const messages: IMessage[] = []; + await waku.store.queryWithPromiseCallback( + [TestDecoder], + async (msgPromise) => { + const msg = await msgPromise; + if (msg) { + messages.push(msg); + } + } + ); + + expect(messages?.length).eq(totalMsgs); + const result = messages?.findIndex((msg) => { + return msg.payload[0]! === 0; + }); + expect(result).to.not.eq(-1); + }); + + it("Callback on promise, aborts when callback returns true", async function () { + this.timeout(15_000); + + const totalMsgs = 20; + + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const desiredMsgs = 14; + const messages: IMessage[] = []; + await waku.store.queryWithPromiseCallback( + [TestDecoder], + async (msgPromise) => { + const msg = await msgPromise; + if (msg) { + messages.push(msg); + } + return messages.length >= desiredMsgs; + }, + { pageSize: 7 } + ); + + expect(messages?.length).eq(desiredMsgs); + }); + + it("Ordered Callback - Forward", async function () { + this.timeout(15_000); + + const totalMsgs = 18; + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + await delay(1); // to ensure each timestamp is unique. + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + messages.push(msg); + }, + { + pageDirection: PageDirection.FORWARD + } + ); + + expect(messages?.length).eq(totalMsgs); + const payloads = messages.map((msg) => msg.payload[0]!); + expect(payloads).to.deep.eq(Array.from(Array(totalMsgs).keys())); + }); + + it("Ordered Callback - Backward", async function () { + this.timeout(15_000); + + const totalMsgs = 18; + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + await delay(1); // to ensure each timestamp is unique. + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + let messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + messages.push(msg); + }, + { + pageDirection: PageDirection.BACKWARD + } + ); + + messages = messages.reverse(); + + expect(messages?.length).eq(totalMsgs); + const payloads = messages.map((msg) => msg.payload![0]!); + expect(payloads).to.deep.eq(Array.from(Array(totalMsgs).keys())); + }); + + it("Generator, with asymmetric & symmetric encrypted messages", async function () { + this.timeout(15_000); + + const asymText = "This message is encrypted for me using asymmetric"; + const asymTopic = "/test/1/asymmetric/proto"; + const symText = + "This message is encrypted for me using symmetric encryption"; + const symTopic = "/test/1/symmetric/proto"; + const clearText = "This is a clear text message for everyone to read"; + const otherText = + "This message is not for and I must not be able to read it"; + + const timestamp = new Date(); + + const asymMsg = { payload: utf8ToBytes(asymText), timestamp }; + const symMsg = { + payload: utf8ToBytes(symText), + timestamp: new Date(timestamp.valueOf() + 1) + }; + const clearMsg = { + payload: utf8ToBytes(clearText), + timestamp: new Date(timestamp.valueOf() + 2) + }; + const otherMsg = { + payload: utf8ToBytes(otherText), + timestamp: new Date(timestamp.valueOf() + 3) + }; + + const privateKey = generatePrivateKey(); + const symKey = generateSymmetricKey(); + const publicKey = getPublicKey(privateKey); + + const eciesEncoder = createEciesEncoder({ + contentTopic: asymTopic, + publicKey + }); + const symEncoder = createSymEncoder({ + contentTopic: symTopic, + symKey + }); + + const otherEncoder = createEciesEncoder({ + contentTopic: TestContentTopic, + publicKey: getPublicKey(generatePrivateKey()) + }); + + const eciesDecoder = createEciesDecoder(asymTopic, privateKey); + const symDecoder = createSymDecoder(symTopic, symKey); + + const [waku1, waku2, nimWakuMultiaddr] = await Promise.all([ + createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }).then((waku) => waku.start().then(() => waku)), + createLightNode({ + staticNoiseKey: NOISE_KEY_2 + }).then((waku) => waku.start().then(() => waku)), + nwaku.getMultiaddrWithId() + ]); + + log("Waku nodes created"); + + await Promise.all([ + waku1.dial(nimWakuMultiaddr), + waku2.dial(nimWakuMultiaddr) + ]); + + log("Waku nodes connected to nwaku"); + + await waitForRemotePeer(waku1, [Protocols.LightPush]); + + log("Sending messages using light push"); + await Promise.all([ + waku1.lightPush.send(eciesEncoder, asymMsg), + waku1.lightPush.send(symEncoder, symMsg), + waku1.lightPush.send(otherEncoder, otherMsg), + waku1.lightPush.send(TestEncoder, clearMsg) + ]); + + await waitForRemotePeer(waku2, [Protocols.Store]); + + const messages: DecodedMessage[] = []; + log("Retrieve messages from store"); + + for await (const msgPromises of waku2.store.queryGenerator([ + eciesDecoder, + symDecoder, + TestDecoder + ])) { + for (const promise of msgPromises) { + const msg = await promise; + if (msg) { + messages.push(msg); + } + } + } + + // Messages are ordered from oldest to latest within a page (1 page query) + expect(bytesToUtf8(messages[0].payload!)).to.eq(asymText); + expect(bytesToUtf8(messages[1].payload!)).to.eq(symText); + expect(bytesToUtf8(messages[2].payload!)).to.eq(clearText); + expect(messages?.length).eq(3); + + !!waku1 && waku1.stop().catch((e) => console.log("Waku failed to stop", e)); + !!waku2 && waku2.stop().catch((e) => console.log("Waku failed to stop", e)); + }); + + it("Ordered callback, using start and end time", async function () { + this.timeout(20000); + + const now = new Date(); + + const startTime = new Date(); + // Set start time 15 seconds in the past + startTime.setTime(now.getTime() - 15 * 1000); + + const message1Timestamp = new Date(); + // Set first message was 10 seconds in the past + message1Timestamp.setTime(now.getTime() - 10 * 1000); + + const message2Timestamp = new Date(); + // Set second message 2 seconds in the past + message2Timestamp.setTime(now.getTime() - 2 * 1000); + const messageTimestamps = [message1Timestamp, message2Timestamp]; + + const endTime = new Date(); + // Set end time 1 second in the past + endTime.setTime(now.getTime() - 1000); + + for (let i = 0; i < 2; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic, + timestamp: messageTimestamps[i] + }) + ) + ).to.be.true; + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const firstMessages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + (msg) => { + if (msg) { + firstMessages.push(msg); + } + }, + { + timeFilter: { startTime, endTime: message1Timestamp } + } + ); + + const bothMessages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + bothMessages.push(msg); + }, + { + timeFilter: { + startTime, + endTime + } + } + ); + + expect(firstMessages?.length).eq(1); + + expect(firstMessages[0].payload![0]!).eq(0); + + expect(bothMessages?.length).eq(2); + }); + + it("Ordered callback, aborts when callback returns true", async function () { + this.timeout(15_000); + + const totalMsgs = 20; + + for (let i = 0; i < totalMsgs; i++) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: TestContentTopic + }) + ) + ).to.be.true; + await delay(1); // to ensure each timestamp is unique. + } + + waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + + const desiredMsgs = 14; + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + messages.push(msg); + return messages.length >= desiredMsgs; + }, + { pageSize: 7 } + ); + + expect(messages?.length).eq(desiredMsgs); + }); +}); diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts new file mode 100644 index 0000000000..455d5a6821 --- /dev/null +++ b/packages/tests/tests/store/utils.ts @@ -0,0 +1,60 @@ +import { Decoder, waitForRemotePeer } from "@waku/core"; +import { IMessage, LightNode, Protocols } from "@waku/interfaces"; +import { createLightNode } from "@waku/sdk"; +import { expect } from "chai"; + +import { delay, NimGoNode, NOISE_KEY_1 } from "../../src"; + +export async function sendMessages( + instance: NimGoNode, + numMessages: number, + contentTopic: string, + pubSubTopic: string +): Promise { + for (let i = 0; i < numMessages; i++) { + expect( + await instance.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([i]), + contentTopic: contentTopic + }), + pubSubTopic + ) + ).to.be.true; + } + await delay(1); // to ensure each timestamp is unique. +} + +export async function processMessages( + instance: LightNode, + decoders: Array, + expectedTopic: string +): Promise { + const localMessages: IMessage[] = []; + let localPromises: Promise[] = []; + for await (const msgPromises of instance.store.queryGenerator(decoders)) { + const _promises = msgPromises.map(async (promise) => { + const msg = await promise; + if (msg) { + localMessages.push(msg); + expect(msg.pubSubTopic).to.eq(expectedTopic); + } + }); + + localPromises = localPromises.concat(_promises); + } + await Promise.all(localPromises); + return localMessages; +} + +export async function startAndConnectLightNode( + instance: NimGoNode +): Promise { + const waku = await createLightNode({ + staticNoiseKey: NOISE_KEY_1 + }); + await waku.start(); + await waku.dial(await instance.getMultiaddrWithId()); + await waitForRemotePeer(waku, [Protocols.Store]); + return waku; +} From c83b976621c3d549aad1f8d960978b892592cd4b Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 29 Sep 2023 19:10:03 +0300 Subject: [PATCH 02/15] refactor store tests --- .../tests/tests/store/multiple_pubsub.spec.ts | 72 ++---- packages/tests/tests/store/store.node.spec.ts | 207 +++--------------- packages/tests/tests/store/utils.ts | 10 +- 3 files changed, 62 insertions(+), 227 deletions(-) diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts index 05367aa46a..34e4dd5618 100644 --- a/packages/tests/tests/store/multiple_pubsub.spec.ts +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -1,15 +1,14 @@ -import { createDecoder, waitForRemotePeer } from "@waku/core"; -import type { IMessage, LightNode } from "@waku/interfaces"; -import { Protocols } from "@waku/interfaces"; -import { createLightNode } from "@waku/sdk"; +import { createDecoder } from "@waku/core"; +import type { LightNode } from "@waku/interfaces"; import { expect } from "chai"; +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + import { - makeLogFileName, - NimGoNode, - NOISE_KEY_1, - tearDownNodes -} from "../../src/index.js"; + processMessages, + sendMessages, + startAndConnectLightNode +} from "./utils.js"; const customPubSubTopic = "/waku/2/custom-dapp/proto"; const TestContentTopic = "/test/1/waku-store/utf8"; @@ -17,8 +16,10 @@ const CustomPubSubTestDecoder = createDecoder( TestContentTopic, customPubSubTopic ); +const totalMsgs = 20; -describe("Waku Store, custom pubsub topic", () => { +describe("Waku Store, custom pubsub topic", function () { + this.timeout(15000); let waku: LightNode; let nwaku: NimGoNode; @@ -27,9 +28,10 @@ describe("Waku Store, custom pubsub topic", () => { nwaku = new NimGoNode(makeLogFileName(this)); await nwaku.startWithRetries({ store: true, - topic: customPubSubTopic, - relay: true + relay: true, + topic: customPubSubTopic }); + await nwaku.ensureSubscriptions([customPubSubTopic]); }); afterEach(async function () { @@ -38,45 +40,13 @@ describe("Waku Store, custom pubsub topic", () => { }); it("Generator, custom pubsub topic", async function () { - this.timeout(15000); - - const totalMsgs = 20; - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }), - customPubSubTopic - ) - ).to.be.true; - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1, - pubSubTopics: [customPubSubTopic] - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); - - const messages: IMessage[] = []; - let promises: Promise[] = []; - for await (const msgPromises of waku.store.queryGenerator([ - CustomPubSubTestDecoder - ])) { - const _promises = msgPromises.map(async (promise) => { - const msg = await promise; - if (msg) { - messages.push(msg); - expect(msg.pubSubTopic).to.eq(customPubSubTopic); - } - }); - - promises = promises.concat(_promises); - } - await Promise.all(promises); + await sendMessages(nwaku, totalMsgs, TestContentTopic, customPubSubTopic); + waku = await startAndConnectLightNode(nwaku, [customPubSubTopic]); + const messages = await processMessages( + waku, + [CustomPubSubTestDecoder], + customPubSubTopic + ); expect(messages?.length).eq(totalMsgs); const result = messages?.findIndex((msg) => { diff --git a/packages/tests/tests/store/store.node.spec.ts b/packages/tests/tests/store/store.node.spec.ts index 4efd1389b9..26616b8b32 100644 --- a/packages/tests/tests/store/store.node.spec.ts +++ b/packages/tests/tests/store/store.node.spec.ts @@ -20,35 +20,27 @@ import { createEncoder as createSymEncoder, generateSymmetricKey } from "@waku/message-encryption/symmetric"; -import { createLightNode } from "@waku/sdk"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; import { expect } from "chai"; -import debug from "debug"; - -import { - delay, - makeLogFileName, - NimGoNode, - NOISE_KEY_1, - NOISE_KEY_2, - tearDownNodes -} from "../../src/index.js"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; import { + log, processMessages, sendMessages, startAndConnectLightNode } from "./utils.js"; -const log = debug("waku:test:store"); - const TestContentTopic = "/test/1/waku-store/utf8"; const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); const TestDecoder = createDecoder(TestContentTopic); +const totalMsgs = 20; -describe.only("Waku Store", function () { +describe("Waku Store", function () { this.timeout(15000); let waku: LightNode; + let waku2: LightNode; let nwaku: NimGoNode; beforeEach(async function () { @@ -59,12 +51,10 @@ describe.only("Waku Store", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes([nwaku], [waku, waku2]); }); it("Generator", async function () { - const totalMsgs = 20; - await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); const messages = await processMessages( @@ -80,7 +70,7 @@ describe.only("Waku Store", function () { expect(result).to.not.eq(-1); }); - it.only("Generator, no message returned", async function () { + it("Generator, no message returned", async function () { waku = await startAndConnectLightNode(nwaku); const messages = await processMessages( waku, @@ -92,26 +82,8 @@ describe.only("Waku Store", function () { }); it("Passing a cursor", async function () { - this.timeout(4_000); - const totalMsgs = 20; - - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: utf8ToBytes(`Message ${i}`), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); const query = waku.store.queryGenerator([TestDecoder]); @@ -148,27 +120,8 @@ describe.only("Waku Store", function () { }); it("Callback on promise", async function () { - this.timeout(15_000); - - const totalMsgs = 15; - - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); const messages: IMessage[] = []; await waku.store.queryWithPromiseCallback( @@ -189,27 +142,8 @@ describe.only("Waku Store", function () { }); it("Callback on promise, aborts when callback returns true", async function () { - this.timeout(15_000); - - const totalMsgs = 20; - - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); const desiredMsgs = 14; const messages: IMessage[] = []; @@ -229,27 +163,8 @@ describe.only("Waku Store", function () { }); it("Ordered Callback - Forward", async function () { - this.timeout(15_000); - - const totalMsgs = 18; - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - await delay(1); // to ensure each timestamp is unique. - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); const messages: IMessage[] = []; await waku.store.queryWithOrderedCallback( @@ -268,27 +183,8 @@ describe.only("Waku Store", function () { }); it("Ordered Callback - Backward", async function () { - this.timeout(15_000); - - const totalMsgs = 18; - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - await delay(1); // to ensure each timestamp is unique. - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); let messages: IMessage[] = []; await waku.store.queryWithOrderedCallback( @@ -309,8 +205,6 @@ describe.only("Waku Store", function () { }); it("Generator, with asymmetric & symmetric encrypted messages", async function () { - this.timeout(15_000); - const asymText = "This message is encrypted for me using asymmetric"; const asymTopic = "/test/1/asymmetric/proto"; const symText = @@ -357,33 +251,25 @@ describe.only("Waku Store", function () { const eciesDecoder = createEciesDecoder(asymTopic, privateKey); const symDecoder = createSymDecoder(symTopic, symKey); - const [waku1, waku2, nimWakuMultiaddr] = await Promise.all([ - createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }).then((waku) => waku.start().then(() => waku)), - createLightNode({ - staticNoiseKey: NOISE_KEY_2 - }).then((waku) => waku.start().then(() => waku)), - nwaku.getMultiaddrWithId() - ]); - - log("Waku nodes created"); + waku = await startAndConnectLightNode(nwaku); + waku2 = await startAndConnectLightNode(nwaku); + const nimWakuMultiaddr = await nwaku.getMultiaddrWithId(); await Promise.all([ - waku1.dial(nimWakuMultiaddr), + waku.dial(nimWakuMultiaddr), waku2.dial(nimWakuMultiaddr) ]); log("Waku nodes connected to nwaku"); - await waitForRemotePeer(waku1, [Protocols.LightPush]); + await waitForRemotePeer(waku, [Protocols.LightPush]); log("Sending messages using light push"); await Promise.all([ - waku1.lightPush.send(eciesEncoder, asymMsg), - waku1.lightPush.send(symEncoder, symMsg), - waku1.lightPush.send(otherEncoder, otherMsg), - waku1.lightPush.send(TestEncoder, clearMsg) + waku.lightPush.send(eciesEncoder, asymMsg), + waku.lightPush.send(symEncoder, symMsg), + waku.lightPush.send(otherEncoder, otherMsg), + waku.lightPush.send(TestEncoder, clearMsg) ]); await waitForRemotePeer(waku2, [Protocols.Store]); @@ -409,14 +295,9 @@ describe.only("Waku Store", function () { expect(bytesToUtf8(messages[1].payload!)).to.eq(symText); expect(bytesToUtf8(messages[2].payload!)).to.eq(clearText); expect(messages?.length).eq(3); - - !!waku1 && waku1.stop().catch((e) => console.log("Waku failed to stop", e)); - !!waku2 && waku2.stop().catch((e) => console.log("Waku failed to stop", e)); }); it("Ordered callback, using start and end time", async function () { - this.timeout(20000); - const now = new Date(); const startTime = new Date(); @@ -436,6 +317,9 @@ describe.only("Waku Store", function () { // Set end time 1 second in the past endTime.setTime(now.getTime() - 1000); + await sendMessages(nwaku, 2, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + for (let i = 0; i < 2; i++) { expect( await nwaku.sendMessage( @@ -448,12 +332,7 @@ describe.only("Waku Store", function () { ).to.be.true; } - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + waku = await startAndConnectLightNode(nwaku); const firstMessages: IMessage[] = []; await waku.store.queryWithOrderedCallback( @@ -490,28 +369,8 @@ describe.only("Waku Store", function () { }); it("Ordered callback, aborts when callback returns true", async function () { - this.timeout(15_000); - - const totalMsgs = 20; - - for (let i = 0; i < totalMsgs; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic - }) - ) - ).to.be.true; - await delay(1); // to ensure each timestamp is unique. - } - - waku = await createLightNode({ - staticNoiseKey: NOISE_KEY_1 - }); - await waku.start(); - await waku.dial(await nwaku.getMultiaddrWithId()); - await waitForRemotePeer(waku, [Protocols.Store]); + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); const desiredMsgs = 14; const messages: IMessage[] = []; diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index 455d5a6821..f875906da3 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -1,10 +1,13 @@ -import { Decoder, waitForRemotePeer } from "@waku/core"; +import { Decoder, DefaultPubSubTopic, waitForRemotePeer } from "@waku/core"; import { IMessage, LightNode, Protocols } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; import { expect } from "chai"; +import debug from "debug"; import { delay, NimGoNode, NOISE_KEY_1 } from "../../src"; +export const log = debug("waku:test:store"); + export async function sendMessages( instance: NimGoNode, numMessages: number, @@ -48,13 +51,16 @@ export async function processMessages( } export async function startAndConnectLightNode( - instance: NimGoNode + instance: NimGoNode, + pubSubTopics: string[] = [DefaultPubSubTopic] ): Promise { const waku = await createLightNode({ + pubSubTopics: pubSubTopics, staticNoiseKey: NOISE_KEY_1 }); await waku.start(); await waku.dial(await instance.getMultiaddrWithId()); await waitForRemotePeer(waku, [Protocols.Store]); + log("Waku node created"); return waku; } From 937147706993f1944d9160064fc9de7308405b11 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 3 Oct 2023 10:05:32 +0300 Subject: [PATCH 03/15] new store tests --- packages/tests/src/message_collector.ts | 17 +++- packages/tests/src/utils.ts | 7 ++ .../tests/tests/filter/subscribe.node.spec.ts | 4 +- .../tests/tests/store/multiple_pubsub.spec.ts | 9 +- packages/tests/tests/store/store.node.spec.ts | 91 +++++++++++++++++-- packages/tests/tests/store/utils.ts | 23 ++++- 6 files changed, 126 insertions(+), 25 deletions(-) create mode 100644 packages/tests/src/utils.ts diff --git a/packages/tests/src/message_collector.ts b/packages/tests/src/message_collector.ts index d8929dc2a4..79ac745138 100644 --- a/packages/tests/src/message_collector.ts +++ b/packages/tests/src/message_collector.ts @@ -1,9 +1,10 @@ import { DecodedMessage, DefaultPubSubTopic } from "@waku/core"; -import { bytesToUtf8 } from "@waku/utils/bytes"; +import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; import { AssertionError, expect } from "chai"; import debug from "debug"; import { MessageRpcResponse } from "./node/interfaces.js"; +import { areUint8ArraysEqual } from "./utils.js"; import { base64ToUtf8, delay, NimGoNode } from "./index.js"; @@ -36,9 +37,17 @@ export class MessageCollector { } hasMessage(topic: string, text: string): boolean { - return this.list.some( - (message) => message.contentTopic === topic && message.payload === text - ); + return this.list.some((message) => { + if (message.contentTopic !== topic) { + return false; + } + if (typeof message.payload === "string") { + return message.payload === text; + } else if (message.payload instanceof Uint8Array) { + return areUint8ArraysEqual(message.payload, utf8ToBytes(text)); + } + return false; + }); } // Type guard to determine if a message is of type MessageRpcResponse diff --git a/packages/tests/src/utils.ts b/packages/tests/src/utils.ts new file mode 100644 index 0000000000..961f5ff440 --- /dev/null +++ b/packages/tests/src/utils.ts @@ -0,0 +1,7 @@ +export function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} diff --git a/packages/tests/tests/filter/subscribe.node.spec.ts b/packages/tests/tests/filter/subscribe.node.spec.ts index 92903e111c..510b94921c 100644 --- a/packages/tests/tests/filter/subscribe.node.spec.ts +++ b/packages/tests/tests/filter/subscribe.node.spec.ts @@ -341,7 +341,7 @@ describe("Waku Filter V2: Subscribe", function () { } // Check if both messages were received - expect(messageCollector.hasMessage(TestContentTopic, "M1")).to.be.true; - expect(messageCollector.hasMessage(newContentTopic, "M2")).to.be.true; + expect(messageCollector.hasMessage(TestContentTopic, "M1")).to.eq(true); + expect(messageCollector.hasMessage(newContentTopic, "M2")).to.eq(true); }); }); diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts index c5f6ac45d2..bf95df9e95 100644 --- a/packages/tests/tests/store/multiple_pubsub.spec.ts +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -17,15 +17,15 @@ import { import { processMessages, sendMessages, - startAndConnectLightNode + startAndConnectLightNode, + TestContentTopic, + TestDecoder, + totalMsgs } from "./utils.js"; -const TestContentTopic = "/test/1/waku-store/utf8"; -const TestDecoder = createDecoder(TestContentTopic); const customContentTopic = "/test/2/waku-store/utf8"; const customPubSubTopic = "/waku/2/custom-dapp/proto"; const customTestDecoder = createDecoder(customContentTopic, customPubSubTopic); -const totalMsgs = 20; describe("Waku Store, custom pubsub topic", function () { this.timeout(15000); @@ -110,6 +110,7 @@ describe("Waku Store, custom pubsub topic", function () { topic: [DefaultPubSubTopic], relay: true }); + await nwaku2.ensureSubscriptions([DefaultPubSubTopic]); const totalMsgs = 10; await sendMessages(nwaku, totalMsgs, customContentTopic, customPubSubTopic); diff --git a/packages/tests/tests/store/store.node.spec.ts b/packages/tests/tests/store/store.node.spec.ts index 26616b8b32..02a962aa9b 100644 --- a/packages/tests/tests/store/store.node.spec.ts +++ b/packages/tests/tests/store/store.node.spec.ts @@ -1,7 +1,6 @@ import { createCursor, createDecoder, - createEncoder, DecodedMessage, DefaultPubSubTopic, PageDirection, @@ -23,20 +22,28 @@ import { import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; import { expect } from "chai"; -import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; +import { + delay, + makeLogFileName, + MessageCollector, + NimGoNode, + tearDownNodes, + TEST_STRING +} from "../../src/index.js"; +import { areUint8ArraysEqual } from "../../src/utils.js"; import { log, + messageText, processMessages, sendMessages, - startAndConnectLightNode + startAndConnectLightNode, + TestContentTopic, + TestDecoder, + TestEncoder, + totalMsgs } from "./utils.js"; -const TestContentTopic = "/test/1/waku-store/utf8"; -const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); -const TestDecoder = createDecoder(TestContentTopic); -const totalMsgs = 20; - describe("Waku Store", function () { this.timeout(15000); let waku: LightNode; @@ -54,7 +61,7 @@ describe("Waku Store", function () { await tearDownNodes([nwaku], [waku, waku2]); }); - it("Generator", async function () { + it("Query generator for multiple messages", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); const messages = await processMessages( @@ -64,13 +71,77 @@ describe("Waku Store", function () { ); expect(messages?.length).eq(totalMsgs); + + // checking that the message with text 0 exists const result = messages?.findIndex((msg) => { return msg.payload[0]! === 0; }); expect(result).to.not.eq(-1); }); - it("Generator, no message returned", async function () { + it("Query generator for multiple messages with different message text format", async function () { + for (const testItem of TEST_STRING) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: utf8ToBytes(testItem["value"]), + contentTopic: TestContentTopic + }), + DefaultPubSubTopic + ) + ).to.be.true; + await delay(1); // to ensure each timestamp is unique. + } + + waku = await startAndConnectLightNode(nwaku); + const messageCollector = new MessageCollector(); + messageCollector.list = await processMessages( + waku, + [TestDecoder], + DefaultPubSubTopic + ); + TEST_STRING.forEach((testItem) => { + expect( + messageCollector.hasMessage(TestContentTopic, testItem["value"]) + ).to.eq(true); + }); + }); + + it("Query generator for multiple messages with different message content topic format", async function () { + for (const testItem of TEST_STRING) { + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: utf8ToBytes(messageText), + contentTopic: testItem["value"] + }), + DefaultPubSubTopic + ) + ).to.be.true; + await delay(1); // to ensure each timestamp is unique. + } + + waku = await startAndConnectLightNode(nwaku); + + let localPromises: Promise[] = []; + for (const testItem of TEST_STRING) { + for await (const msgPromises of waku.store.queryGenerator([ + createDecoder(testItem["value"]) + ])) { + const _promises = msgPromises.map(async (promise) => { + const msg = await promise; + if (msg) { + areUint8ArraysEqual(msg.payload, utf8ToBytes(messageText)); + } + }); + + localPromises = localPromises.concat(_promises); + } + await Promise.all(localPromises); + } + }); + + it("Query generator, no message returned", async function () { waku = await startAndConnectLightNode(nwaku); const messages = await processMessages( waku, diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index f875906da3..40af75a5e6 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -1,5 +1,12 @@ -import { Decoder, DefaultPubSubTopic, waitForRemotePeer } from "@waku/core"; -import { IMessage, LightNode, Protocols } from "@waku/interfaces"; +import { + createDecoder, + createEncoder, + DecodedMessage, + Decoder, + DefaultPubSubTopic, + waitForRemotePeer +} from "@waku/core"; +import { LightNode, Protocols } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; import { expect } from "chai"; import debug from "debug"; @@ -8,6 +15,12 @@ import { delay, NimGoNode, NOISE_KEY_1 } from "../../src"; export const log = debug("waku:test:store"); +export const TestContentTopic = "/test/1/waku-store/utf8"; +export const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); +export const TestDecoder = createDecoder(TestContentTopic); +export const totalMsgs = 20; +export const messageText = "Store Push works!"; + export async function sendMessages( instance: NimGoNode, numMessages: number, @@ -24,16 +37,16 @@ export async function sendMessages( pubSubTopic ) ).to.be.true; + await delay(1); // to ensure each timestamp is unique. } - await delay(1); // to ensure each timestamp is unique. } export async function processMessages( instance: LightNode, decoders: Array, expectedTopic: string -): Promise { - const localMessages: IMessage[] = []; +): Promise { + const localMessages: DecodedMessage[] = []; let localPromises: Promise[] = []; for await (const msgPromises of instance.store.queryGenerator(decoders)) { const _promises = msgPromises.map(async (promise) => { From 6778eb380999156a417387a868248996e06a3671 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 3 Oct 2023 12:46:22 +0300 Subject: [PATCH 04/15] new test --- .../tests/tests/store/multiple_pubsub.spec.ts | 13 ++----- packages/tests/tests/store/store.node.spec.ts | 38 ++++++++++++++++++- packages/tests/tests/store/utils.ts | 6 +++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts index bf95df9e95..8ce77d22f6 100644 --- a/packages/tests/tests/store/multiple_pubsub.spec.ts +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -1,8 +1,4 @@ -import { - createDecoder, - DefaultPubSubTopic, - waitForRemotePeer -} from "@waku/core"; +import { DefaultPubSubTopic, waitForRemotePeer } from "@waku/core"; import type { IMessage, LightNode } from "@waku/interfaces"; import { createLightNode, Protocols } from "@waku/sdk"; import { expect } from "chai"; @@ -15,6 +11,9 @@ import { } from "../../src/index.js"; import { + customContentTopic, + customPubSubTopic, + customTestDecoder, processMessages, sendMessages, startAndConnectLightNode, @@ -23,10 +22,6 @@ import { totalMsgs } from "./utils.js"; -const customContentTopic = "/test/2/waku-store/utf8"; -const customPubSubTopic = "/waku/2/custom-dapp/proto"; -const customTestDecoder = createDecoder(customContentTopic, customPubSubTopic); - describe("Waku Store, custom pubsub topic", function () { this.timeout(15000); let waku: LightNode; diff --git a/packages/tests/tests/store/store.node.spec.ts b/packages/tests/tests/store/store.node.spec.ts index 02a962aa9b..699ce83ebd 100644 --- a/packages/tests/tests/store/store.node.spec.ts +++ b/packages/tests/tests/store/store.node.spec.ts @@ -33,6 +33,7 @@ import { import { areUint8ArraysEqual } from "../../src/utils.js"; import { + customContentTopic, log, messageText, processMessages, @@ -100,6 +101,8 @@ describe("Waku Store", function () { [TestDecoder], DefaultPubSubTopic ); + + // checking that all message sent were retrieved TEST_STRING.forEach((testItem) => { expect( messageCollector.hasMessage(TestContentTopic, testItem["value"]) @@ -107,7 +110,36 @@ describe("Waku Store", function () { }); }); - it("Query generator for multiple messages with different message content topic format", async function () { + it("Query generator for multiple messages with multiple decoders", async function () { + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: utf8ToBytes("M1"), + contentTopic: TestContentTopic + }), + DefaultPubSubTopic + ); + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: utf8ToBytes("M2"), + contentTopic: customContentTopic + }), + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + const secondDecoder = createDecoder(customContentTopic, DefaultPubSubTopic); + + const messageCollector = new MessageCollector(); + messageCollector.list = await processMessages( + waku, + [TestDecoder, secondDecoder], + DefaultPubSubTopic + ); + expect(messageCollector.hasMessage(TestContentTopic, "M1")).to.eq(true); + expect(messageCollector.hasMessage(customContentTopic, "M2")).to.eq(true); + }); + + it("Query generator for multiple messages with different content topic format", async function () { for (const testItem of TEST_STRING) { expect( await nwaku.sendMessage( @@ -131,7 +163,9 @@ describe("Waku Store", function () { const _promises = msgPromises.map(async (promise) => { const msg = await promise; if (msg) { - areUint8ArraysEqual(msg.payload, utf8ToBytes(messageText)); + expect( + areUint8ArraysEqual(msg.payload, utf8ToBytes(messageText)) + ).to.eq(true); } }); diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index 40af75a5e6..1b731fa8ac 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -18,6 +18,12 @@ export const log = debug("waku:test:store"); export const TestContentTopic = "/test/1/waku-store/utf8"; export const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); export const TestDecoder = createDecoder(TestContentTopic); +export const customContentTopic = "/test/2/waku-store/utf8"; +export const customPubSubTopic = "/waku/2/custom-dapp/proto"; +export const customTestDecoder = createDecoder( + customContentTopic, + customPubSubTopic +); export const totalMsgs = 20; export const messageText = "Store Push works!"; From 8fbb9c0c1b2674ce4484bff43138a3438f6d041d Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Wed, 4 Oct 2023 15:40:50 +0300 Subject: [PATCH 05/15] new store tests --- .../{index.spec.ts => index.node.spec.ts} | 0 .../tests/tests/store/cursor.node.spec.ts | 147 ++++++++++++ .../tests/store/error_handling.node.spec.ts | 224 ++++++++++++++++++ ...{store.node.spec.ts => index.node.spec.ts} | 134 ++--------- .../tests/tests/store/multiple_pubsub.spec.ts | 12 +- packages/tests/tests/store/order.node.spec.ts | 129 ++++++++++ packages/tests/tests/store/utils.ts | 28 ++- 7 files changed, 539 insertions(+), 135 deletions(-) rename packages/tests/tests/light-push/{index.spec.ts => index.node.spec.ts} (100%) create mode 100644 packages/tests/tests/store/cursor.node.spec.ts create mode 100644 packages/tests/tests/store/error_handling.node.spec.ts rename packages/tests/tests/store/{store.node.spec.ts => index.node.spec.ts} (74%) create mode 100644 packages/tests/tests/store/order.node.spec.ts diff --git a/packages/tests/tests/light-push/index.spec.ts b/packages/tests/tests/light-push/index.node.spec.ts similarity index 100% rename from packages/tests/tests/light-push/index.spec.ts rename to packages/tests/tests/light-push/index.node.spec.ts diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts new file mode 100644 index 0000000000..1b883917ad --- /dev/null +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -0,0 +1,147 @@ +import { createCursor, DecodedMessage, DefaultPubSubTopic } from "@waku/core"; +import type { LightNode } from "@waku/interfaces"; +import { bytesToUtf8 } from "@waku/utils/bytes"; +import { expect } from "chai"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + customPubSubTopic, + sendMessages, + startAndConnectLightNode, + TestContentTopic, + TestDecoder, + totalMsgs +} from "./utils.js"; + +describe("Waku Store, cursor", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + [ + [2, 4], + [0, 20], + [10, 40], + [19, 20], + [19, 50], + [110, 120] + ].forEach(([cursorIndex, messageCount]) => { + it(`Passing a valid cursor at ${cursorIndex} index when there are ${messageCount} messages`, async function () { + await sendMessages( + nwaku, + messageCount, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + // messages in reversed order (first message at last index) + const messages: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder])) { + for await (const msg of page.reverse()) { + messages.push(msg as DecodedMessage); + } + } + + // create cursor to extract messages after the cursorIndex + const cursor = await createCursor(messages[cursorIndex]); + + // cursor.digest = new Uint8Array([]); + + const messagesAfterCursor: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + if (msg) { + messagesAfterCursor.push(msg as DecodedMessage); + } + } + } + + expect(messages.length).be.eql(messageCount); + expect(messagesAfterCursor.length).be.eql(messageCount - cursorIndex - 1); + if (cursorIndex == messages.length - 1) { + // in this case the cursor will return nothin because it points at the end of the list + expect(messagesAfterCursor).be.eql([]); + } else { + expect(bytesToUtf8(messagesAfterCursor[0].payload)).to.be.eq( + bytesToUtf8(messages[cursorIndex + 1].payload) + ); + expect( + bytesToUtf8( + messagesAfterCursor[messagesAfterCursor.length - 1].payload + ) + ).to.be.eq(bytesToUtf8(messages[messages.length - 1].payload)); + } + }); + }); + + it("Passing cursor with wrong message digest", async function () { + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + + const messages: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder])) { + for await (const msg of page.reverse()) { + messages.push(msg as DecodedMessage); + } + } + const cursor = await createCursor(messages[5]); + + // setting a wrong digest + cursor.digest = new Uint8Array([]); + + const messagesAfterCursor: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + if (msg) { + messagesAfterCursor.push(msg as DecodedMessage); + } + } + } + expect(messagesAfterCursor.length).be.eql(0); + }); + + // Skipped because of strange results. Generator retrieves messages even if cursor is using a different customPubSubTopic. + // My guess is that pubsubTopic is not used. Need to confirm + it("Passing cursor with wrong pubSubTopic", async function () { + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + + const messages: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder])) { + for await (const msg of page.reverse()) { + messages.push(msg as DecodedMessage); + } + } + const cursor = await createCursor(messages[5], customPubSubTopic); + + const messagesAfterCursor: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + if (msg) { + messagesAfterCursor.push(msg as DecodedMessage); + } + } + } + expect(messagesAfterCursor.length).be.eql(0); + }); +}); diff --git a/packages/tests/tests/store/error_handling.node.spec.ts b/packages/tests/tests/store/error_handling.node.spec.ts new file mode 100644 index 0000000000..8cbf0e44d6 --- /dev/null +++ b/packages/tests/tests/store/error_handling.node.spec.ts @@ -0,0 +1,224 @@ +import { DefaultPubSubTopic } from "@waku/core"; +import { IMessage, type LightNode } from "@waku/interfaces"; +import { expect } from "chai"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + customPubSubTopic, + customTestDecoder, + processQueriedMessages, + startAndConnectLightNode, + TestDecoder +} from "./utils.js"; + +describe("Waku Store, Error Handling", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + waku = await startAndConnectLightNode(nwaku); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + it("Query Generator, Wrong PubSubTopic", async function () { + try { + for await (const msgPromises of waku.store.queryGenerator([ + customTestDecoder + ])) { + msgPromises; + } + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + `PubSub topic ${customPubSubTopic} has not been configured on this instance. Configured topics are: ${DefaultPubSubTopic}` + ) + ) { + throw err; + } + } + }); + + it("Query Generator, Multiple PubSubTopics", async function () { + try { + for await (const msgPromises of waku.store.queryGenerator([ + TestDecoder, + customTestDecoder + ])) { + msgPromises; + } + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + "API does not support querying multiple pubsub topics at once" + ) + ) { + throw err; + } + } + }); + + it("Query Generator, No Decoder", async function () { + try { + for await (const msgPromises of waku.store.queryGenerator([])) { + msgPromises; + } + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes("No decoders provided") + ) { + throw err; + } + } + }); + + it("Query Generator, No message returned", async function () { + const messages = await processQueriedMessages( + waku, + [TestDecoder], + DefaultPubSubTopic + ); + expect(messages?.length).eq(0); + }); + + it("Query with Ordered Callback, Wrong PubSubTopic", async function () { + try { + await waku.store.queryWithOrderedCallback( + [customTestDecoder], + async () => {} + ); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + `PubSub topic ${customPubSubTopic} has not been configured on this instance. Configured topics are: ${DefaultPubSubTopic}` + ) + ) { + throw err; + } + } + }); + + it("Query with Ordered Callback, Multiple PubSubTopics", async function () { + try { + await waku.store.queryWithOrderedCallback( + [TestDecoder, customTestDecoder], + async () => {} + ); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + "API does not support querying multiple pubsub topics at once" + ) + ) { + throw err; + } + } + }); + + it("Query with Ordered Callback, No Decoder", async function () { + try { + await waku.store.queryWithOrderedCallback([], async () => {}); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes("No decoders provided") + ) { + throw err; + } + } + }); + + it("Query with Ordered Callback, No message returned", async function () { + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback([TestDecoder], async (msg) => { + messages.push(msg); + }); + expect(messages?.length).eq(0); + }); + + it("Query with Promise Callback, Wrong PubSubTopic", async function () { + try { + await waku.store.queryWithPromiseCallback( + [customTestDecoder], + async () => {} + ); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + `PubSub topic ${customPubSubTopic} has not been configured on this instance. Configured topics are: ${DefaultPubSubTopic}` + ) + ) { + throw err; + } + } + }); + + it("Query with Promise Callback, Multiple PubSubTopics", async function () { + try { + await waku.store.queryWithPromiseCallback( + [TestDecoder, customTestDecoder], + async () => {} + ); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + "API does not support querying multiple pubsub topics at once" + ) + ) { + throw err; + } + } + }); + + it("Query with Promise Callback, No Decoder", async function () { + try { + await waku.store.queryWithPromiseCallback([], async () => {}); + throw new Error("QueryGenerator was successful but was expected to fail"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes("No decoders provided") + ) { + throw err; + } + } + }); + + it("Query with Promise Callback, No message returned", async function () { + const messages: IMessage[] = []; + await waku.store.queryWithPromiseCallback( + [TestDecoder], + async (msgPromise) => { + const msg = await msgPromise; + if (msg) { + messages.push(msg); + } + } + ); + expect(messages?.length).eq(0); + }); +}); diff --git a/packages/tests/tests/store/store.node.spec.ts b/packages/tests/tests/store/index.node.spec.ts similarity index 74% rename from packages/tests/tests/store/store.node.spec.ts rename to packages/tests/tests/store/index.node.spec.ts index 699ce83ebd..4eddb598bc 100644 --- a/packages/tests/tests/store/store.node.spec.ts +++ b/packages/tests/tests/store/index.node.spec.ts @@ -1,9 +1,7 @@ import { - createCursor, createDecoder, DecodedMessage, DefaultPubSubTopic, - PageDirection, waitForRemotePeer } from "@waku/core"; import type { IMessage, LightNode } from "@waku/interfaces"; @@ -36,7 +34,7 @@ import { customContentTopic, log, messageText, - processMessages, + processQueriedMessages, sendMessages, startAndConnectLightNode, TestContentTopic, @@ -45,6 +43,8 @@ import { totalMsgs } from "./utils.js"; +const secondDecoder = createDecoder(customContentTopic, DefaultPubSubTopic); + describe("Waku Store", function () { this.timeout(15000); let waku: LightNode; @@ -55,6 +55,7 @@ describe("Waku Store", function () { this.timeout(15000); nwaku = new NimGoNode(makeLogFileName(this)); await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); }); afterEach(async function () { @@ -65,7 +66,7 @@ describe("Waku Store", function () { it("Query generator for multiple messages", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); - const messages = await processMessages( + const messages = await processQueriedMessages( waku, [TestDecoder], DefaultPubSubTopic @@ -96,7 +97,7 @@ describe("Waku Store", function () { waku = await startAndConnectLightNode(nwaku); const messageCollector = new MessageCollector(); - messageCollector.list = await processMessages( + messageCollector.list = await processQueriedMessages( waku, [TestDecoder], DefaultPubSubTopic @@ -127,10 +128,8 @@ describe("Waku Store", function () { ); waku = await startAndConnectLightNode(nwaku); - const secondDecoder = createDecoder(customContentTopic, DefaultPubSubTopic); - const messageCollector = new MessageCollector(); - messageCollector.list = await processMessages( + messageCollector.list = await processQueriedMessages( waku, [TestDecoder, secondDecoder], DefaultPubSubTopic @@ -155,73 +154,17 @@ describe("Waku Store", function () { waku = await startAndConnectLightNode(nwaku); - let localPromises: Promise[] = []; for (const testItem of TEST_STRING) { - for await (const msgPromises of waku.store.queryGenerator([ + for await (const query of waku.store.queryGenerator([ createDecoder(testItem["value"]) ])) { - const _promises = msgPromises.map(async (promise) => { - const msg = await promise; - if (msg) { - expect( - areUint8ArraysEqual(msg.payload, utf8ToBytes(messageText)) - ).to.eq(true); - } - }); - - localPromises = localPromises.concat(_promises); - } - await Promise.all(localPromises); - } - }); - - it("Query generator, no message returned", async function () { - waku = await startAndConnectLightNode(nwaku); - const messages = await processMessages( - waku, - [TestDecoder], - DefaultPubSubTopic - ); - - expect(messages?.length).eq(0); - }); - - it("Passing a cursor", async function () { - await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); - waku = await startAndConnectLightNode(nwaku); - - const query = waku.store.queryGenerator([TestDecoder]); - - // messages in reversed order (first message at last index) - const messages: DecodedMessage[] = []; - for await (const page of query) { - for await (const msg of page.reverse()) { - messages.push(msg as DecodedMessage); + for await (const msg of query) { + expect( + areUint8ArraysEqual(msg!.payload, utf8ToBytes(messageText)) + ).to.eq(true); + } } } - - // index 2 would mean the third last message sent - const cursorIndex = 2; - - // create cursor to extract messages after the 3rd index - const cursor = await createCursor(messages[cursorIndex]); - - const messagesAfterCursor: DecodedMessage[] = []; - for await (const page of waku.store.queryGenerator([TestDecoder], { - cursor - })) { - for await (const msg of page.reverse()) { - messagesAfterCursor.push(msg as DecodedMessage); - } - } - - const testMessage = messagesAfterCursor[0]; - - expect(messages.length).be.eq(totalMsgs); - - expect(bytesToUtf8(testMessage.payload)).to.be.eq( - bytesToUtf8(messages[cursorIndex + 1].payload) - ); }); it("Callback on promise", async function () { @@ -267,48 +210,6 @@ describe("Waku Store", function () { expect(messages?.length).eq(desiredMsgs); }); - it("Ordered Callback - Forward", async function () { - await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); - waku = await startAndConnectLightNode(nwaku); - - const messages: IMessage[] = []; - await waku.store.queryWithOrderedCallback( - [TestDecoder], - async (msg) => { - messages.push(msg); - }, - { - pageDirection: PageDirection.FORWARD - } - ); - - expect(messages?.length).eq(totalMsgs); - const payloads = messages.map((msg) => msg.payload[0]!); - expect(payloads).to.deep.eq(Array.from(Array(totalMsgs).keys())); - }); - - it("Ordered Callback - Backward", async function () { - await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); - waku = await startAndConnectLightNode(nwaku); - - let messages: IMessage[] = []; - await waku.store.queryWithOrderedCallback( - [TestDecoder], - async (msg) => { - messages.push(msg); - }, - { - pageDirection: PageDirection.BACKWARD - } - ); - - messages = messages.reverse(); - - expect(messages?.length).eq(totalMsgs); - const payloads = messages.map((msg) => msg.payload![0]!); - expect(payloads).to.deep.eq(Array.from(Array(totalMsgs).keys())); - }); - it("Generator, with asymmetric & symmetric encrypted messages", async function () { const asymText = "This message is encrypted for me using asymmetric"; const asymTopic = "/test/1/asymmetric/proto"; @@ -382,15 +283,14 @@ describe("Waku Store", function () { const messages: DecodedMessage[] = []; log("Retrieve messages from store"); - for await (const msgPromises of waku2.store.queryGenerator([ + for await (const query of waku2.store.queryGenerator([ eciesDecoder, symDecoder, TestDecoder ])) { - for (const promise of msgPromises) { - const msg = await promise; + for await (const msg of query) { if (msg) { - messages.push(msg); + messages.push(msg as DecodedMessage); } } } @@ -437,8 +337,6 @@ describe("Waku Store", function () { ).to.be.true; } - waku = await startAndConnectLightNode(nwaku); - const firstMessages: IMessage[] = []; await waku.store.queryWithOrderedCallback( [TestDecoder], diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts index 8ce77d22f6..48a0d39fb4 100644 --- a/packages/tests/tests/store/multiple_pubsub.spec.ts +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -14,7 +14,7 @@ import { customContentTopic, customPubSubTopic, customTestDecoder, - processMessages, + processQueriedMessages, sendMessages, startAndConnectLightNode, TestContentTopic, @@ -47,7 +47,7 @@ describe("Waku Store, custom pubsub topic", function () { it("Generator, custom pubsub topic", async function () { await sendMessages(nwaku, totalMsgs, customContentTopic, customPubSubTopic); waku = await startAndConnectLightNode(nwaku, [customPubSubTopic]); - const messages = await processMessages( + const messages = await processQueriedMessages( waku, [customTestDecoder], customPubSubTopic @@ -72,7 +72,7 @@ describe("Waku Store, custom pubsub topic", function () { DefaultPubSubTopic ]); - const customMessages = await processMessages( + const customMessages = await processQueriedMessages( waku, [customTestDecoder], customPubSubTopic @@ -83,7 +83,7 @@ describe("Waku Store, custom pubsub topic", function () { }); expect(result1).to.not.eq(-1); - const testMessages = await processMessages( + const testMessages = await processQueriedMessages( waku, [TestDecoder], DefaultPubSubTopic @@ -128,12 +128,12 @@ describe("Waku Store, custom pubsub topic", function () { customMessages.length != totalMsgs || testMessages.length != totalMsgs ) { - customMessages = await processMessages( + customMessages = await processQueriedMessages( waku, [customTestDecoder], customPubSubTopic ); - testMessages = await processMessages( + testMessages = await processQueriedMessages( waku, [TestDecoder], DefaultPubSubTopic diff --git a/packages/tests/tests/store/order.node.spec.ts b/packages/tests/tests/store/order.node.spec.ts new file mode 100644 index 0000000000..4b5db460ca --- /dev/null +++ b/packages/tests/tests/store/order.node.spec.ts @@ -0,0 +1,129 @@ +import { DecodedMessage, DefaultPubSubTopic, PageDirection } from "@waku/core"; +import type { IMessage, LightNode } from "@waku/interfaces"; +import { expect } from "chai"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + chunkAndReverseArray, + sendMessages, + startAndConnectLightNode, + TestContentTopic, + TestDecoder, + totalMsgs +} from "./utils.js"; + +describe("Waku Store, order", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { + it(`Query Generator - ${pageDirection}`, async function () { + await sendMessages( + nwaku, + totalMsgs, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + for await (const query of waku.store.queryGenerator([TestDecoder], { + pageDirection: pageDirection + })) { + for await (const msg of query) { + if (msg) { + messages.push(msg as DecodedMessage); + } + } + } + + let expectedPayloads = Array.from(Array(totalMsgs).keys()); + if (pageDirection === PageDirection.BACKWARD) { + expectedPayloads = chunkAndReverseArray(expectedPayloads, 10); + } + + expect(messages?.length).eq(totalMsgs); + const payloads = messages.map((msg) => msg.payload[0]!); + expect(payloads).to.deep.eq(expectedPayloads); + }); + }); + + [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { + it(`Promise Callback - ${pageDirection}`, async function () { + await sendMessages( + nwaku, + totalMsgs, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + await waku.store.queryWithPromiseCallback( + [TestDecoder], + async (msgPromise) => { + const msg = await msgPromise; + if (msg) { + messages.push(msg); + } + }, + { + pageDirection: pageDirection + } + ); + + let expectedPayloads = Array.from(Array(totalMsgs).keys()); + if (pageDirection === PageDirection.BACKWARD) { + expectedPayloads = chunkAndReverseArray(expectedPayloads, 10); + } + + expect(messages?.length).eq(totalMsgs); + const payloads = messages.map((msg) => msg.payload[0]!); + expect(payloads).to.deep.eq(expectedPayloads); + }); + }); + + [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { + it(`Ordered Callback - ${pageDirection}`, async function () { + await sendMessages( + nwaku, + totalMsgs, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + messages.push(msg); + }, + { + pageDirection: pageDirection + } + ); + + if (pageDirection === PageDirection.BACKWARD) { + messages.reverse(); + } + expect(messages?.length).eq(totalMsgs); + const payloads = messages.map((msg) => msg.payload[0]!); + expect(payloads).to.deep.eq(Array.from(Array(totalMsgs).keys())); + }); + }); +}); diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index 1b731fa8ac..1580e5b447 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -47,25 +47,20 @@ export async function sendMessages( } } -export async function processMessages( +export async function processQueriedMessages( instance: LightNode, decoders: Array, - expectedTopic: string + expectedTopic?: string ): Promise { const localMessages: DecodedMessage[] = []; - let localPromises: Promise[] = []; - for await (const msgPromises of instance.store.queryGenerator(decoders)) { - const _promises = msgPromises.map(async (promise) => { - const msg = await promise; + for await (const query of instance.store.queryGenerator(decoders)) { + for await (const msg of query) { if (msg) { - localMessages.push(msg); expect(msg.pubSubTopic).to.eq(expectedTopic); + localMessages.push(msg as DecodedMessage); } - }); - - localPromises = localPromises.concat(_promises); + } } - await Promise.all(localPromises); return localMessages; } @@ -83,3 +78,14 @@ export async function startAndConnectLightNode( log("Waku node created"); return waku; } + +export function chunkAndReverseArray( + arr: number[], + chunkSize: number +): number[] { + const result: number[] = []; + for (let i = 0; i < arr.length; i += chunkSize) { + result.push(...arr.slice(i, i + chunkSize).reverse()); + } + return result.reverse(); +} From 6c35a51f678a30c17131f89134daaee6d4156892 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Wed, 4 Oct 2023 16:00:55 +0300 Subject: [PATCH 06/15] new store tests --- packages/tests/tests/store/cursor.node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index 1b883917ad..9fc2518aea 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -120,7 +120,7 @@ describe("Waku Store, cursor", function () { // Skipped because of strange results. Generator retrieves messages even if cursor is using a different customPubSubTopic. // My guess is that pubsubTopic is not used. Need to confirm - it("Passing cursor with wrong pubSubTopic", async function () { + it.skip("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); From c52b895c1e7b14b4fb00689377916e6b542e038c Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Thu, 5 Oct 2023 19:06:37 +0300 Subject: [PATCH 07/15] new store tests --- .../tests/store/error_handling.node.spec.ts | 2 +- packages/tests/tests/store/index.node.spec.ts | 82 +++---------- .../tests/tests/store/page_size.node.spec.ts | 94 +++++++++++++++ .../tests/tests/store/sorting.node.spec.ts | 110 ++++++++++++++++++ .../tests/store/time_filter.node.spec.ts | 88 ++++++++++++++ packages/tests/tests/store/utils.ts | 6 + 6 files changed, 312 insertions(+), 70 deletions(-) create mode 100644 packages/tests/tests/store/page_size.node.spec.ts create mode 100644 packages/tests/tests/store/sorting.node.spec.ts create mode 100644 packages/tests/tests/store/time_filter.node.spec.ts diff --git a/packages/tests/tests/store/error_handling.node.spec.ts b/packages/tests/tests/store/error_handling.node.spec.ts index 8cbf0e44d6..286f32b77c 100644 --- a/packages/tests/tests/store/error_handling.node.spec.ts +++ b/packages/tests/tests/store/error_handling.node.spec.ts @@ -12,7 +12,7 @@ import { TestDecoder } from "./utils.js"; -describe("Waku Store, Error Handling", function () { +describe("Waku Store, error handling", function () { this.timeout(15000); let waku: LightNode; let nwaku: NimGoNode; diff --git a/packages/tests/tests/store/index.node.spec.ts b/packages/tests/tests/store/index.node.spec.ts index 4eddb598bc..8cb95fc19e 100644 --- a/packages/tests/tests/store/index.node.spec.ts +++ b/packages/tests/tests/store/index.node.spec.ts @@ -302,75 +302,6 @@ describe("Waku Store", function () { expect(messages?.length).eq(3); }); - it("Ordered callback, using start and end time", async function () { - const now = new Date(); - - const startTime = new Date(); - // Set start time 15 seconds in the past - startTime.setTime(now.getTime() - 15 * 1000); - - const message1Timestamp = new Date(); - // Set first message was 10 seconds in the past - message1Timestamp.setTime(now.getTime() - 10 * 1000); - - const message2Timestamp = new Date(); - // Set second message 2 seconds in the past - message2Timestamp.setTime(now.getTime() - 2 * 1000); - const messageTimestamps = [message1Timestamp, message2Timestamp]; - - const endTime = new Date(); - // Set end time 1 second in the past - endTime.setTime(now.getTime() - 1000); - - await sendMessages(nwaku, 2, TestContentTopic, DefaultPubSubTopic); - waku = await startAndConnectLightNode(nwaku); - - for (let i = 0; i < 2; i++) { - expect( - await nwaku.sendMessage( - NimGoNode.toMessageRpcQuery({ - payload: new Uint8Array([i]), - contentTopic: TestContentTopic, - timestamp: messageTimestamps[i] - }) - ) - ).to.be.true; - } - - const firstMessages: IMessage[] = []; - await waku.store.queryWithOrderedCallback( - [TestDecoder], - (msg) => { - if (msg) { - firstMessages.push(msg); - } - }, - { - timeFilter: { startTime, endTime: message1Timestamp } - } - ); - - const bothMessages: IMessage[] = []; - await waku.store.queryWithOrderedCallback( - [TestDecoder], - async (msg) => { - bothMessages.push(msg); - }, - { - timeFilter: { - startTime, - endTime - } - } - ); - - expect(firstMessages?.length).eq(1); - - expect(firstMessages[0].payload![0]!).eq(0); - - expect(bothMessages?.length).eq(2); - }); - it("Ordered callback, aborts when callback returns true", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); @@ -388,4 +319,17 @@ describe("Waku Store", function () { expect(messages?.length).eq(desiredMsgs); }); + + it("Query generator for 2000 messages", async function () { + this.timeout(40000); + await sendMessages(nwaku, 2000, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + const messages = await processQueriedMessages( + waku, + [TestDecoder], + DefaultPubSubTopic + ); + + expect(messages?.length).eq(2000); + }); }); diff --git a/packages/tests/tests/store/page_size.node.spec.ts b/packages/tests/tests/store/page_size.node.spec.ts new file mode 100644 index 0000000000..cf0f3fa3df --- /dev/null +++ b/packages/tests/tests/store/page_size.node.spec.ts @@ -0,0 +1,94 @@ +import { DefaultPubSubTopic } from "@waku/core"; +import type { LightNode } from "@waku/interfaces"; +import { expect } from "chai"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + sendMessages, + startAndConnectLightNode, + TestContentTopic, + TestDecoder +} from "./utils.js"; + +describe("Waku Store, page size", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + [ + [0, 30], + [1, 4], + [3, 20], + [10, 10], + [11, 10], + [19, 20], + [110, 120] + ].forEach(([pageSize, messageCount]) => { + it(`Passing page size ${pageSize} when there are ${messageCount} messages`, async function () { + await sendMessages( + nwaku, + messageCount, + TestContentTopic, + DefaultPubSubTopic + ); + + // Determine effectivePageSize for test expectations + let effectivePageSize = pageSize; + if (pageSize === 0) { + effectivePageSize = 20; + } else if (pageSize > 100) { + effectivePageSize = 100; + } + + waku = await startAndConnectLightNode(nwaku); + let messagesRetrieved = 0; + for await (const query of waku.store.queryGenerator([TestDecoder], { + pageSize: pageSize + })) { + // Calculate expected page size + const expectedPageSize = Math.min( + effectivePageSize, + messageCount - messagesRetrieved + ); + expect(query.length).eq(expectedPageSize); + + for await (const msg of query) { + if (msg) { + messagesRetrieved++; + } + } + } + + expect(messagesRetrieved).eq(messageCount); + }); + }); + + it("Default pageSize", async function () { + await sendMessages(nwaku, 20, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + + let messagesRetrieved = 0; + for await (const query of waku.store.queryGenerator([TestDecoder])) { + expect(query.length).eq(10); + for await (const msg of query) { + if (msg) { + messagesRetrieved++; + } + } + } + expect(messagesRetrieved).eq(20); + }); +}); diff --git a/packages/tests/tests/store/sorting.node.spec.ts b/packages/tests/tests/store/sorting.node.spec.ts new file mode 100644 index 0000000000..e4c7c1be5b --- /dev/null +++ b/packages/tests/tests/store/sorting.node.spec.ts @@ -0,0 +1,110 @@ +import { DecodedMessage, DefaultPubSubTopic, PageDirection } from "@waku/core"; +import type { IMessage, LightNode } from "@waku/interfaces"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + sendMessages, + startAndConnectLightNode, + TestContentTopic, + TestDecoder, + totalMsgs +} from "./utils.js"; + +describe("Waku Store, sorting", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { + it(`Query Generator sorting by timestamp while page direction is ${pageDirection}`, async function () { + await sendMessages( + nwaku, + totalMsgs, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + for await (const query of waku.store.queryGenerator([TestDecoder], { + pageDirection: PageDirection.FORWARD + })) { + const page: IMessage[] = []; + for await (const msg of query) { + if (msg) { + page.push(msg as DecodedMessage); + } + } + // Extract timestamps + const timestamps = page.map( + (msg) => msg.timestamp as unknown as bigint + ); + // Check if timestamps are sorted + for (let i = 1; i < timestamps.length; i++) { + if (timestamps[i] < timestamps[i - 1]) { + throw new Error( + `Messages are not sorted by timestamp. Found out of order at index ${i}` + ); + } + } + } + }); + }); + + [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { + it(`Ordered Callback sorting by timestamp while page direction is ${pageDirection}`, async function () { + await sendMessages( + nwaku, + totalMsgs, + TestContentTopic, + DefaultPubSubTopic + ); + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + async (msg) => { + messages.push(msg); + }, + { + pageDirection: pageDirection + } + ); + // Extract timestamps + const timestamps = messages.map( + (msg) => msg.timestamp as unknown as bigint + ); + // Check if timestamps are sorted + for (let i = 1; i < timestamps.length; i++) { + if ( + pageDirection === PageDirection.FORWARD && + timestamps[i] < timestamps[i - 1] + ) { + throw new Error( + `Messages are not sorted by timestamp in FORWARD direction. Found out of order at index ${i}` + ); + } else if ( + pageDirection === PageDirection.BACKWARD && + timestamps[i] > timestamps[i - 1] + ) { + throw new Error( + `Messages are not sorted by timestamp in BACKWARD direction. Found out of order at index ${i}` + ); + } + } + }); + }); +}); diff --git a/packages/tests/tests/store/time_filter.node.spec.ts b/packages/tests/tests/store/time_filter.node.spec.ts new file mode 100644 index 0000000000..b0dc9ccfc4 --- /dev/null +++ b/packages/tests/tests/store/time_filter.node.spec.ts @@ -0,0 +1,88 @@ +import type { IMessage, LightNode } from "@waku/interfaces"; +import { expect } from "chai"; + +import { makeLogFileName, NimGoNode, tearDownNodes } from "../../src/index.js"; + +import { + adjustDate, + startAndConnectLightNode, + TestContentTopic, + TestDecoder +} from "./utils.js"; + +describe("Waku Store, time filter", function () { + this.timeout(15000); + let waku: LightNode; + let nwaku: NimGoNode; + + beforeEach(async function () { + this.timeout(15000); + nwaku = new NimGoNode(makeLogFileName(this)); + await nwaku.startWithRetries({ store: true, lightpush: true, relay: true }); + await nwaku.ensureSubscriptions(); + }); + + afterEach(async function () { + this.timeout(15000); + await tearDownNodes([nwaku], [waku]); + }); + + [ + [-10000, -10, 10], + [-10000, 1, 4], + [-10000, -2, -1], + [-10000, 0, 1000], + [-10000, -1000, 0], + [10000, 4, 1], + [10000, -10, 10] + ].forEach(([msgTimeAdjustment, startTime, endTime]) => { + it(`msgTime: ${adjustDate( + new Date(), + msgTimeAdjustment + )}, startTime: ${adjustDate( + adjustDate(new Date(), msgTimeAdjustment), + startTime + )}, endTime: ${adjustDate( + adjustDate(new Date(), msgTimeAdjustment), + endTime + )}`, async function () { + const msgTimestamp = adjustDate(new Date(), msgTimeAdjustment); + + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([0]), + contentTopic: TestContentTopic, + timestamp: msgTimestamp + }) + ) + ).to.be.true; + + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + (msg) => { + if (msg) { + messages.push(msg); + } + }, + { + timeFilter: { + startTime: adjustDate(msgTimestamp, startTime), + endTime: adjustDate(msgTimestamp, endTime) + } + } + ); + + // in this context 0 is the messageTimestamp + if ((startTime > 0 && endTime > 0) || (startTime < 0 && endTime < 0)) { + expect(messages.length).eq(0); + } else { + expect(messages.length).eq(1); + expect(messages[0].payload![0]!).eq(0); + } + }); + }); +}); diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index 1580e5b447..d4ccac15ce 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -89,3 +89,9 @@ export function chunkAndReverseArray( } return result.reverse(); } + +export const adjustDate = (baseDate: Date, adjustMs: number): Date => { + const adjusted = new Date(baseDate); + adjusted.setTime(adjusted.getTime() + adjustMs); + return adjusted; +}; From 5519877b5e8a48ecacd31b01aa930e227cee205d Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 6 Oct 2023 11:24:10 +0300 Subject: [PATCH 08/15] adjustments --- .../tests/tests/store/cursor.node.spec.ts | 30 +++++--- .../tests/tests/store/page_size.node.spec.ts | 8 ++- .../tests/store/time_filter.node.spec.ts | 68 +++++++++++++------ 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index 9fc2518aea..e12bae4531 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -106,20 +106,32 @@ describe("Waku Store, cursor", function () { cursor.digest = new Uint8Array([]); const messagesAfterCursor: DecodedMessage[] = []; - for await (const page of waku.store.queryGenerator([TestDecoder], { - cursor - })) { - for await (const msg of page.reverse()) { - if (msg) { - messagesAfterCursor.push(msg as DecodedMessage); + try { + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + if (msg) { + messagesAfterCursor.push(msg as DecodedMessage); + } } } + expect(messagesAfterCursor.length).to.eql(0); + } catch (error) { + if ( + nwaku.type() === "go-waku" && + typeof error === "string" && + error.includes("History response contains an Error: INVALID_CURSOR") + ) { + return; + } + throw error instanceof Error + ? new Error(`Unexpected error: ${error.message}`) + : error; } - expect(messagesAfterCursor.length).be.eql(0); }); - // Skipped because of strange results. Generator retrieves messages even if cursor is using a different customPubSubTopic. - // My guess is that pubsubTopic is not used. Need to confirm + // PubsubTopic is ignored in the cursor. Needs fixing so it throws an error if it doesn't match with Decoder it.skip("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); diff --git a/packages/tests/tests/store/page_size.node.spec.ts b/packages/tests/tests/store/page_size.node.spec.ts index cf0f3fa3df..2e2a552b07 100644 --- a/packages/tests/tests/store/page_size.node.spec.ts +++ b/packages/tests/tests/store/page_size.node.spec.ts @@ -29,7 +29,7 @@ describe("Waku Store, page size", function () { }); [ - [0, 30], + [0, 110], [1, 4], [3, 20], [10, 10], @@ -48,7 +48,11 @@ describe("Waku Store, page size", function () { // Determine effectivePageSize for test expectations let effectivePageSize = pageSize; if (pageSize === 0) { - effectivePageSize = 20; + if (nwaku.type() == "go-waku") { + effectivePageSize = 100; + } else { + effectivePageSize = 20; + } } else if (pageSize > 100) { effectivePageSize = 100; } diff --git a/packages/tests/tests/store/time_filter.node.spec.ts b/packages/tests/tests/store/time_filter.node.spec.ts index b0dc9ccfc4..ff78fc61e5 100644 --- a/packages/tests/tests/store/time_filter.node.spec.ts +++ b/packages/tests/tests/store/time_filter.node.spec.ts @@ -28,26 +28,19 @@ describe("Waku Store, time filter", function () { }); [ - [-10000, -10, 10], - [-10000, 1, 4], - [-10000, -2, -1], - [-10000, 0, 1000], - [-10000, -1000, 0], - [10000, 4, 1], - [10000, -10, 10] - ].forEach(([msgTimeAdjustment, startTime, endTime]) => { - it(`msgTime: ${adjustDate( - new Date(), - msgTimeAdjustment - )}, startTime: ${adjustDate( - adjustDate(new Date(), msgTimeAdjustment), - startTime - )}, endTime: ${adjustDate( - adjustDate(new Date(), msgTimeAdjustment), - endTime - )}`, async function () { - const msgTimestamp = adjustDate(new Date(), msgTimeAdjustment); - + [-19000, -10, 10], + [-19000, 1, 4], + [-19000, -2, -1], + [-19000, 0, 1000], + [-19000, -1000, 0], + [19000, 4, 1], + [19000, -10010, -9990], + [19000, -10, 10] + ].forEach(([msgTime, startTime, endTime]) => { + it(`msgTime: ${msgTime} ms from now, startTime: ${ + msgTime + startTime + }, endTime: ${msgTime + endTime}`, async function () { + const msgTimestamp = adjustDate(new Date(), msgTime); expect( await nwaku.sendMessage( NimGoNode.toMessageRpcQuery({ @@ -85,4 +78,39 @@ describe("Waku Store, time filter", function () { } }); }); + + [-20000, 40000].forEach((msgTime) => { + it(`Timestamp too far from node time: ${msgTime} ms from now`, async function () { + const msgTimestamp = adjustDate(new Date(), msgTime); + expect( + await nwaku.sendMessage( + NimGoNode.toMessageRpcQuery({ + payload: new Uint8Array([0]), + contentTopic: TestContentTopic, + timestamp: msgTimestamp + }) + ) + ).to.be.true; + + waku = await startAndConnectLightNode(nwaku); + + const messages: IMessage[] = []; + await waku.store.queryWithOrderedCallback( + [TestDecoder], + (msg) => { + if (msg) { + messages.push(msg); + } + }, + { + timeFilter: { + startTime: adjustDate(msgTimestamp, -1000), + endTime: adjustDate(msgTimestamp, 1000) + } + } + ); + + expect(messages.length).eq(0); + }); + }); }); From 0e5ff3e13d5e5869d0833359ad106fea51b53c5c Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 6 Oct 2023 12:45:00 +0300 Subject: [PATCH 09/15] add final tests --- packages/tests/.mocharc.json | 4 +- .../tests/tests/store/cursor.node.spec.ts | 43 +++++++++++++++++-- .../tests/store/time_filter.node.spec.ts | 13 +++--- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/packages/tests/.mocharc.json b/packages/tests/.mocharc.json index e277cf89b1..01f73e807e 100644 --- a/packages/tests/.mocharc.json +++ b/packages/tests/.mocharc.json @@ -6,6 +6,6 @@ "experimental-specifier-resolution=node", "loader=ts-node/esm" ], - "exit": true, - "retries": 3 + "exit": true + // "retries": 3 } diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index e12bae4531..b8cd09d34b 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -17,6 +17,7 @@ import { describe("Waku Store, cursor", function () { this.timeout(15000); let waku: LightNode; + let waku2: LightNode; let nwaku: NimGoNode; beforeEach(async function () { @@ -28,7 +29,7 @@ describe("Waku Store, cursor", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes([nwaku], [waku, waku2]); }); [ @@ -59,8 +60,6 @@ describe("Waku Store, cursor", function () { // create cursor to extract messages after the cursorIndex const cursor = await createCursor(messages[cursorIndex]); - // cursor.digest = new Uint8Array([]); - const messagesAfterCursor: DecodedMessage[] = []; for await (const page of waku.store.queryGenerator([TestDecoder], { cursor @@ -90,6 +89,44 @@ describe("Waku Store, cursor", function () { }); }); + it("Reusing cursor across nodes", async function () { + await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); + waku = await startAndConnectLightNode(nwaku); + waku2 = await startAndConnectLightNode(nwaku); + + // messages in reversed order (first message at last index) + const messages: DecodedMessage[] = []; + for await (const page of waku.store.queryGenerator([TestDecoder])) { + for await (const msg of page.reverse()) { + messages.push(msg as DecodedMessage); + } + } + + // create cursor to extract messages after the cursorIndex + const cursor = await createCursor(messages[5]); + + // query node2 with the cursor from node1 + const messagesAfterCursor: DecodedMessage[] = []; + for await (const page of waku2.store.queryGenerator([TestDecoder], { + cursor + })) { + for await (const msg of page.reverse()) { + if (msg) { + messagesAfterCursor.push(msg as DecodedMessage); + } + } + } + + expect(messages.length).be.eql(totalMsgs); + expect(messagesAfterCursor.length).be.eql(totalMsgs - 6); + expect(bytesToUtf8(messagesAfterCursor[0].payload)).to.be.eq( + bytesToUtf8(messages[6].payload) + ); + expect( + bytesToUtf8(messagesAfterCursor[messagesAfterCursor.length - 1].payload) + ).to.be.eq(bytesToUtf8(messages[messages.length - 1].payload)); + }); + it("Passing cursor with wrong message digest", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); diff --git a/packages/tests/tests/store/time_filter.node.spec.ts b/packages/tests/tests/store/time_filter.node.spec.ts index ff78fc61e5..8214454672 100644 --- a/packages/tests/tests/store/time_filter.node.spec.ts +++ b/packages/tests/tests/store/time_filter.node.spec.ts @@ -31,11 +31,10 @@ describe("Waku Store, time filter", function () { [-19000, -10, 10], [-19000, 1, 4], [-19000, -2, -1], - [-19000, 0, 1000], + // [-19000, 0, 1000], // skipped because it fails on gowaku [-19000, -1000, 0], - [19000, 4, 1], - [19000, -10010, -9990], - [19000, -10, 10] + [19000, -10, 10], // message in the future + [-19000, 10, -10] // startTime is newer than endTime ].forEach(([msgTime, startTime, endTime]) => { it(`msgTime: ${msgTime} ms from now, startTime: ${ msgTime + startTime @@ -70,7 +69,11 @@ describe("Waku Store, time filter", function () { ); // in this context 0 is the messageTimestamp - if ((startTime > 0 && endTime > 0) || (startTime < 0 && endTime < 0)) { + if ( + (startTime > 0 && endTime > 0) || + (startTime < 0 && endTime < 0) || + startTime > endTime + ) { expect(messages.length).eq(0); } else { expect(messages.length).eq(1); From 95dbac3c57c1bb9ba1208bc0a2a579110914d000 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 6 Oct 2023 13:34:30 +0300 Subject: [PATCH 10/15] self review --- packages/tests/.mocharc.json | 4 ++-- packages/tests/tests/store/index.node.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/tests/.mocharc.json b/packages/tests/.mocharc.json index 01f73e807e..e277cf89b1 100644 --- a/packages/tests/.mocharc.json +++ b/packages/tests/.mocharc.json @@ -6,6 +6,6 @@ "experimental-specifier-resolution=node", "loader=ts-node/esm" ], - "exit": true - // "retries": 3 + "exit": true, + "retries": 3 } diff --git a/packages/tests/tests/store/index.node.spec.ts b/packages/tests/tests/store/index.node.spec.ts index 8cb95fc19e..44f3fb023a 100644 --- a/packages/tests/tests/store/index.node.spec.ts +++ b/packages/tests/tests/store/index.node.spec.ts @@ -45,7 +45,7 @@ import { const secondDecoder = createDecoder(customContentTopic, DefaultPubSubTopic); -describe("Waku Store", function () { +describe("Waku Store, general", function () { this.timeout(15000); let waku: LightNode; let waku2: LightNode; From b0d658c4a1e99d9a0a2da29babd87a16a2eea378 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Mon, 9 Oct 2023 14:52:40 +0300 Subject: [PATCH 11/15] add comments for known issues --- packages/tests/tests/store/cursor.node.spec.ts | 3 ++- packages/tests/tests/store/page_size.node.spec.ts | 1 + packages/tests/tests/store/time_filter.node.spec.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index b8cd09d34b..39bd940f8e 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -153,6 +153,7 @@ describe("Waku Store, cursor", function () { } } } + // Should return same as go-waku. Raised bug: https://github.com/waku-org/nwaku/issues/2117 expect(messagesAfterCursor.length).to.eql(0); } catch (error) { if ( @@ -168,7 +169,7 @@ describe("Waku Store, cursor", function () { } }); - // PubsubTopic is ignored in the cursor. Needs fixing so it throws an error if it doesn't match with Decoder + // PubsubTopic is ignored in the cursor: https://github.com/waku-org/js-waku/pull/1640 it.skip("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); diff --git a/packages/tests/tests/store/page_size.node.spec.ts b/packages/tests/tests/store/page_size.node.spec.ts index 2e2a552b07..cad9496412 100644 --- a/packages/tests/tests/store/page_size.node.spec.ts +++ b/packages/tests/tests/store/page_size.node.spec.ts @@ -80,6 +80,7 @@ describe("Waku Store, page size", function () { }); }); + // Possible issue here because pageSize differs across implementations it("Default pageSize", async function () { await sendMessages(nwaku, 20, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); diff --git a/packages/tests/tests/store/time_filter.node.spec.ts b/packages/tests/tests/store/time_filter.node.spec.ts index 8214454672..a56cb54e28 100644 --- a/packages/tests/tests/store/time_filter.node.spec.ts +++ b/packages/tests/tests/store/time_filter.node.spec.ts @@ -31,7 +31,7 @@ describe("Waku Store, time filter", function () { [-19000, -10, 10], [-19000, 1, 4], [-19000, -2, -1], - // [-19000, 0, 1000], // skipped because it fails on gowaku + // [-19000, 0, 1000], // skipped for now because it fails on gowaku which returns messages > startTime [-19000, -1000, 0], [19000, -10, 10], // message in the future [-19000, 10, -10] // startTime is newer than endTime From 1d1c7a28a899b2477ead9b5c302e7bc6e2e313fe Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 10 Oct 2023 08:51:41 +0300 Subject: [PATCH 12/15] updated wrong cursor test --- .../tests/tests/store/cursor.node.spec.ts | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index 39bd940f8e..d823492374 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -169,8 +169,7 @@ describe("Waku Store, cursor", function () { } }); - // PubsubTopic is ignored in the cursor: https://github.com/waku-org/js-waku/pull/1640 - it.skip("Passing cursor with wrong pubSubTopic", async function () { + it("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); @@ -180,18 +179,25 @@ describe("Waku Store, cursor", function () { messages.push(msg as DecodedMessage); } } - const cursor = await createCursor(messages[5], customPubSubTopic); + messages[5].pubSubTopic = customPubSubTopic; + const cursor = await createCursor(messages[5]); - const messagesAfterCursor: DecodedMessage[] = []; - for await (const page of waku.store.queryGenerator([TestDecoder], { - cursor - })) { - for await (const msg of page.reverse()) { - if (msg) { - messagesAfterCursor.push(msg as DecodedMessage); - } + try { + for await (const page of waku.store.queryGenerator([TestDecoder], { + cursor + })) { + page; + } + throw new Error("Cursor with wrong pubsubtopic was accepted"); + } catch (err) { + if ( + !(err instanceof Error) || + !err.message.includes( + `Cursor pubsub topic (${customPubSubTopic}) does not match decoder pubsub topic (${DefaultPubSubTopic})` + ) + ) { + throw err; } } - expect(messagesAfterCursor.length).be.eql(0); }); }); From c9389e3d0784941cd5247afd1e57d94292c3c3e7 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 10 Oct 2023 09:03:44 +0300 Subject: [PATCH 13/15] improve teardown nodes --- packages/tests/src/teardown.ts | 11 +++++++---- .../tests/tests/filter/multiple_pubsub.node.spec.ts | 2 +- packages/tests/tests/filter/ping.node.spec.ts | 2 +- packages/tests/tests/filter/push.node.spec.ts | 2 +- packages/tests/tests/filter/subscribe.node.spec.ts | 2 +- packages/tests/tests/filter/unsubscribe.node.spec.ts | 2 +- packages/tests/tests/light-push/index.node.spec.ts | 2 +- .../tests/light-push/multiple_pubsub.node.spec.ts | 2 +- packages/tests/tests/store/cursor.node.spec.ts | 4 ++-- .../tests/tests/store/error_handling.node.spec.ts | 2 +- packages/tests/tests/store/index.node.spec.ts | 2 +- packages/tests/tests/store/multiple_pubsub.spec.ts | 2 +- packages/tests/tests/store/order.node.spec.ts | 2 +- packages/tests/tests/store/page_size.node.spec.ts | 2 +- packages/tests/tests/store/sorting.node.spec.ts | 2 +- packages/tests/tests/store/time_filter.node.spec.ts | 2 +- 16 files changed, 23 insertions(+), 20 deletions(-) diff --git a/packages/tests/src/teardown.ts b/packages/tests/src/teardown.ts index c4605d9c43..e68782999e 100644 --- a/packages/tests/src/teardown.ts +++ b/packages/tests/src/teardown.ts @@ -7,10 +7,13 @@ import { NimGoNode } from "./index.js"; const log = debug("waku:test"); export async function tearDownNodes( - nwakuNodes: NimGoNode[], - wakuNodes: LightNode[] + nwakuNodes: NimGoNode | NimGoNode[], + wakuNodes: LightNode | LightNode[] ): Promise { - const stopNwakuNodes = nwakuNodes.map(async (nwaku) => { + const nNodes = Array.isArray(nwakuNodes) ? nwakuNodes : [nwakuNodes]; + const wNodes = Array.isArray(wakuNodes) ? wakuNodes : [wakuNodes]; + + const stopNwakuNodes = nNodes.map(async (nwaku) => { if (nwaku) { await pRetry( async () => { @@ -26,7 +29,7 @@ export async function tearDownNodes( } }); - const stopWakuNodes = wakuNodes.map(async (waku) => { + const stopWakuNodes = wNodes.map(async (waku) => { if (waku) { await pRetry( async () => { diff --git a/packages/tests/tests/filter/multiple_pubsub.node.spec.ts b/packages/tests/tests/filter/multiple_pubsub.node.spec.ts index 724afeba2f..dd0ed193fd 100644 --- a/packages/tests/tests/filter/multiple_pubsub.node.spec.ts +++ b/packages/tests/tests/filter/multiple_pubsub.node.spec.ts @@ -51,7 +51,7 @@ describe("Waku Filter V2: Multiple PubSubtopics", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku, nwaku2], [waku]); + await tearDownNodes([nwaku, nwaku2], waku); }); it("Subscribe and receive messages on custom pubsubtopic", async function () { diff --git a/packages/tests/tests/filter/ping.node.spec.ts b/packages/tests/tests/filter/ping.node.spec.ts index 5b1e4abc38..843133e5d8 100644 --- a/packages/tests/tests/filter/ping.node.spec.ts +++ b/packages/tests/tests/filter/ping.node.spec.ts @@ -30,7 +30,7 @@ describe("Waku Filter V2: Ping", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); it("Ping on subscribed peer", async function () { diff --git a/packages/tests/tests/filter/push.node.spec.ts b/packages/tests/tests/filter/push.node.spec.ts index b04bf68011..69be141025 100644 --- a/packages/tests/tests/filter/push.node.spec.ts +++ b/packages/tests/tests/filter/push.node.spec.ts @@ -38,7 +38,7 @@ describe("Waku Filter V2: FilterPush", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); TEST_STRING.forEach((testItem) => { diff --git a/packages/tests/tests/filter/subscribe.node.spec.ts b/packages/tests/tests/filter/subscribe.node.spec.ts index 8fde8bd217..ebb2f072a8 100644 --- a/packages/tests/tests/filter/subscribe.node.spec.ts +++ b/packages/tests/tests/filter/subscribe.node.spec.ts @@ -49,7 +49,7 @@ describe("Waku Filter V2: Subscribe", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku, nwaku2], [waku]); + await tearDownNodes([nwaku, nwaku2], waku); }); it("Subscribe and receive messages via lightPush", async function () { diff --git a/packages/tests/tests/filter/unsubscribe.node.spec.ts b/packages/tests/tests/filter/unsubscribe.node.spec.ts index 61a3f85b5e..b79d3e313e 100644 --- a/packages/tests/tests/filter/unsubscribe.node.spec.ts +++ b/packages/tests/tests/filter/unsubscribe.node.spec.ts @@ -35,7 +35,7 @@ describe("Waku Filter V2: Unsubscribe", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); it("Unsubscribe 1 topic - node subscribed to 1 topic", async function () { diff --git a/packages/tests/tests/light-push/index.node.spec.ts b/packages/tests/tests/light-push/index.node.spec.ts index 84e8d606e2..4e47a4b451 100644 --- a/packages/tests/tests/light-push/index.node.spec.ts +++ b/packages/tests/tests/light-push/index.node.spec.ts @@ -36,7 +36,7 @@ describe("Waku Light Push", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); TEST_STRING.forEach((testItem) => { diff --git a/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts b/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts index 49d1573cb1..05f8f5068a 100644 --- a/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts +++ b/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts @@ -48,7 +48,7 @@ describe("Waku Light Push : Multiple PubSubtopics", function () { this.afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku, nwaku2], [waku]); + await tearDownNodes([nwaku, nwaku2], waku); }); it("Push message on custom pubSubTopic", async function () { diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index d823492374..7b7b5343d5 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -29,7 +29,7 @@ describe("Waku Store, cursor", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku, waku2]); + await tearDownNodes(nwaku, [waku, waku2]); }); [ @@ -169,7 +169,7 @@ describe("Waku Store, cursor", function () { } }); - it("Passing cursor with wrong pubSubTopic", async function () { + it.only("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku); diff --git a/packages/tests/tests/store/error_handling.node.spec.ts b/packages/tests/tests/store/error_handling.node.spec.ts index 286f32b77c..37fa6d8971 100644 --- a/packages/tests/tests/store/error_handling.node.spec.ts +++ b/packages/tests/tests/store/error_handling.node.spec.ts @@ -27,7 +27,7 @@ describe("Waku Store, error handling", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); it("Query Generator, Wrong PubSubTopic", async function () { diff --git a/packages/tests/tests/store/index.node.spec.ts b/packages/tests/tests/store/index.node.spec.ts index 44f3fb023a..964151a637 100644 --- a/packages/tests/tests/store/index.node.spec.ts +++ b/packages/tests/tests/store/index.node.spec.ts @@ -60,7 +60,7 @@ describe("Waku Store, general", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku, waku2]); + await tearDownNodes(nwaku, [waku, waku2]); }); it("Query generator for multiple messages", async function () { diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts index 48a0d39fb4..f64e80be7e 100644 --- a/packages/tests/tests/store/multiple_pubsub.spec.ts +++ b/packages/tests/tests/store/multiple_pubsub.spec.ts @@ -41,7 +41,7 @@ describe("Waku Store, custom pubsub topic", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku, nwaku2], [waku]); + await tearDownNodes([nwaku, nwaku2], waku); }); it("Generator, custom pubsub topic", async function () { diff --git a/packages/tests/tests/store/order.node.spec.ts b/packages/tests/tests/store/order.node.spec.ts index 4b5db460ca..98f34b4bcb 100644 --- a/packages/tests/tests/store/order.node.spec.ts +++ b/packages/tests/tests/store/order.node.spec.ts @@ -27,7 +27,7 @@ describe("Waku Store, order", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { diff --git a/packages/tests/tests/store/page_size.node.spec.ts b/packages/tests/tests/store/page_size.node.spec.ts index cad9496412..4f1b466a10 100644 --- a/packages/tests/tests/store/page_size.node.spec.ts +++ b/packages/tests/tests/store/page_size.node.spec.ts @@ -25,7 +25,7 @@ describe("Waku Store, page size", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); [ diff --git a/packages/tests/tests/store/sorting.node.spec.ts b/packages/tests/tests/store/sorting.node.spec.ts index e4c7c1be5b..b7d232cec6 100644 --- a/packages/tests/tests/store/sorting.node.spec.ts +++ b/packages/tests/tests/store/sorting.node.spec.ts @@ -25,7 +25,7 @@ describe("Waku Store, sorting", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); [PageDirection.FORWARD, PageDirection.BACKWARD].forEach((pageDirection) => { diff --git a/packages/tests/tests/store/time_filter.node.spec.ts b/packages/tests/tests/store/time_filter.node.spec.ts index a56cb54e28..f53ff23b8f 100644 --- a/packages/tests/tests/store/time_filter.node.spec.ts +++ b/packages/tests/tests/store/time_filter.node.spec.ts @@ -24,7 +24,7 @@ describe("Waku Store, time filter", function () { afterEach(async function () { this.timeout(15000); - await tearDownNodes([nwaku], [waku]); + await tearDownNodes(nwaku, waku); }); [ From aa93a441d57ba3b049c0a5b35d21fdcd43d112ee Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 10 Oct 2023 09:15:36 +0300 Subject: [PATCH 14/15] use loadash instead of custom function --- package-lock.json | 23 +++++++++++++++++-- packages/tests/package.json | 2 ++ packages/tests/src/message_collector.ts | 4 ++-- packages/tests/src/utils.ts | 7 ------ packages/tests/tests/store/index.node.spec.ts | 6 ++--- 5 files changed, 27 insertions(+), 15 deletions(-) delete mode 100644 packages/tests/src/utils.ts diff --git a/package-lock.json b/package-lock.json index 799a09ac41..6a2f998419 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3961,6 +3961,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/lodash": { + "version": "4.14.199", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.199.tgz", + "integrity": "sha512-Vrjz5N5Ia4SEzWWgIVwnHNEnb1UE1XMkvY5DGXrAeOGE9imk0hgTHh5GyDjLDJi9OTCn9oo9dXH1uToK1VRfrg==", + "dev": true + }, "node_modules/@types/markdown-it": { "version": "12.2.3", "dev": true, @@ -13571,7 +13577,8 @@ }, "node_modules/lodash": { "version": "4.17.21", - "license": "MIT" + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash-es": { "version": "4.17.21", @@ -26134,6 +26141,7 @@ "chai-as-promised": "^7.1.1", "debug": "^4.3.4", "dockerode": "^3.3.5", + "lodash": "^4.17.21", "p-retry": "^6.1.0", "p-timeout": "^6.1.0", "portfinder": "^1.0.32", @@ -26144,6 +26152,7 @@ "@libp2p/bootstrap": "^9.0.2", "@types/chai": "^4.3.5", "@types/dockerode": "^3.3.19", + "@types/lodash": "^4.14.199", "@types/mocha": "^10.0.1", "@types/sinon": "^10.0.16", "@types/tail": "^2.2.1", @@ -28698,6 +28707,12 @@ "version": "3.0.3", "dev": true }, + "@types/lodash": { + "version": "4.14.199", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.199.tgz", + "integrity": "sha512-Vrjz5N5Ia4SEzWWgIVwnHNEnb1UE1XMkvY5DGXrAeOGE9imk0hgTHh5GyDjLDJi9OTCn9oo9dXH1uToK1VRfrg==", + "dev": true + }, "@types/markdown-it": { "version": "12.2.3", "dev": true, @@ -29322,6 +29337,7 @@ "@libp2p/peer-id": "^3.0.2", "@types/chai": "^4.3.5", "@types/dockerode": "^3.3.19", + "@types/lodash": "^4.14.199", "@types/mocha": "^10.0.1", "@types/sinon": "^10.0.16", "@types/tail": "^2.2.1", @@ -29342,6 +29358,7 @@ "dockerode": "^3.3.5", "interface-datastore": "^8.2.5", "libp2p": "^0.46.12", + "lodash": "^4.17.21", "mocha": "^10.2.0", "npm-run-all": "^4.1.5", "p-retry": "^6.1.0", @@ -34746,7 +34763,9 @@ } }, "lodash": { - "version": "4.17.21" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash-es": { "version": "4.17.21" diff --git a/packages/tests/package.json b/packages/tests/package.json index 330e46d299..b5c5a844c8 100644 --- a/packages/tests/package.json +++ b/packages/tests/package.json @@ -60,6 +60,7 @@ "chai-as-promised": "^7.1.1", "debug": "^4.3.4", "dockerode": "^3.3.5", + "lodash": "^4.17.21", "p-retry": "^6.1.0", "p-timeout": "^6.1.0", "portfinder": "^1.0.32", @@ -70,6 +71,7 @@ "@libp2p/bootstrap": "^9.0.2", "@types/chai": "^4.3.5", "@types/dockerode": "^3.3.19", + "@types/lodash": "^4.14.199", "@types/mocha": "^10.0.1", "@types/sinon": "^10.0.16", "@types/tail": "^2.2.1", diff --git a/packages/tests/src/message_collector.ts b/packages/tests/src/message_collector.ts index 79ac745138..5b26e56615 100644 --- a/packages/tests/src/message_collector.ts +++ b/packages/tests/src/message_collector.ts @@ -2,9 +2,9 @@ import { DecodedMessage, DefaultPubSubTopic } from "@waku/core"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; import { AssertionError, expect } from "chai"; import debug from "debug"; +import isEqual from "lodash/isEqual"; import { MessageRpcResponse } from "./node/interfaces.js"; -import { areUint8ArraysEqual } from "./utils.js"; import { base64ToUtf8, delay, NimGoNode } from "./index.js"; @@ -44,7 +44,7 @@ export class MessageCollector { if (typeof message.payload === "string") { return message.payload === text; } else if (message.payload instanceof Uint8Array) { - return areUint8ArraysEqual(message.payload, utf8ToBytes(text)); + return isEqual(message.payload, utf8ToBytes(text)); } return false; }); diff --git a/packages/tests/src/utils.ts b/packages/tests/src/utils.ts deleted file mode 100644 index 961f5ff440..0000000000 --- a/packages/tests/src/utils.ts +++ /dev/null @@ -1,7 +0,0 @@ -export function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { - if (a.length !== b.length) return false; - for (let i = 0; i < a.length; i++) { - if (a[i] !== b[i]) return false; - } - return true; -} diff --git a/packages/tests/tests/store/index.node.spec.ts b/packages/tests/tests/store/index.node.spec.ts index 964151a637..f519535a72 100644 --- a/packages/tests/tests/store/index.node.spec.ts +++ b/packages/tests/tests/store/index.node.spec.ts @@ -19,6 +19,7 @@ import { } from "@waku/message-encryption/symmetric"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; import { expect } from "chai"; +import isEqual from "lodash/isEqual"; import { delay, @@ -28,7 +29,6 @@ import { tearDownNodes, TEST_STRING } from "../../src/index.js"; -import { areUint8ArraysEqual } from "../../src/utils.js"; import { customContentTopic, @@ -159,9 +159,7 @@ describe("Waku Store, general", function () { createDecoder(testItem["value"]) ])) { for await (const msg of query) { - expect( - areUint8ArraysEqual(msg!.payload, utf8ToBytes(messageText)) - ).to.eq(true); + expect(isEqual(msg!.payload, utf8ToBytes(messageText))).to.eq(true); } } } From 6f8cf039dfff3f30341c51f53cb9137951c49e1e Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 10 Oct 2023 09:23:51 +0300 Subject: [PATCH 15/15] remove .only --- packages/tests/tests/store/cursor.node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index 7b7b5343d5..20b0f20ec9 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -169,7 +169,7 @@ describe("Waku Store, cursor", function () { } }); - it.only("Passing cursor with wrong pubSubTopic", async function () { + it("Passing cursor with wrong pubSubTopic", async function () { await sendMessages(nwaku, totalMsgs, TestContentTopic, DefaultPubSubTopic); waku = await startAndConnectLightNode(nwaku);