test: Fix flaky ephemeral test and general improvement

The messages were sent at the same time over light push so there was
no strong order preservation from the behaviour.
Correction: order does not matter, just check that messages aren't
present.

Messages were only checked for `ephemeral` being false + one test was
doing several checks.
Correction: split the test and use light push + filter to check
ephemeral field value preservation.

packages/core/src/lib/waku_message/version_0.spec.ts

@@ -24,7 +24,7 @@ describe("Waku Message version 0", function () {
     );
   });
 
-  it("Ephemeral", async function () {
+  it("Ephemeral field set to true", async function () {
     await fc.assert(
       fc.asyncProperty(fc.uint8Array({ minLength: 1 }), async (payload) => {
         const encoder = new EncoderV0(TestContentTopic, true);

packages/tests/tests/ephemeral.node.spec.ts

@@ -0,0 +1,269 @@
+import { bytesToUtf8, utf8ToBytes } from "@waku/byte-utils";
+import { waitForRemotePeer } from "@waku/core/lib/wait_for_remote_peer";
+import { DecoderV0, EncoderV0 } from "@waku/core/lib/waku_message/version_0";
+import { createFullNode, createLightNode } from "@waku/create";
+import { DecodedMessage, Protocols, WakuLight } from "@waku/interfaces";
+import {
+  AsymDecoder,
+  AsymEncoder,
+  generatePrivateKey,
+  generateSymmetricKey,
+  getPublicKey,
+  SymDecoder,
+  SymEncoder,
+} from "@waku/message-encryption";
+import { expect } from "chai";
+import debug from "debug";
+
+import { makeLogFileName, NOISE_KEY_1, NOISE_KEY_2, Nwaku } from "../src";
+import { delay } from "../src/delay";
+
+const log = debug("waku:test:ephemeral");
+
+const TestContentTopic = "/test/1/ephemeral/utf8";
+const TestEncoder = new EncoderV0(TestContentTopic);
+const TestDecoder = new DecoderV0(TestContentTopic);
+
+describe("Waku Message Ephemeral field", () => {
+  let waku: WakuLight;
+  let nwaku: Nwaku;
+
+  afterEach(async function () {
+    !!nwaku && nwaku.stop();
+    !!waku && waku.stop().catch((e) => console.log("Waku failed to stop", e));
+  });
+
+  beforeEach(async function () {
+    this.timeout(15000);
+    nwaku = new Nwaku(makeLogFileName(this));
+    await nwaku.start({ filter: true, lightpush: true, store: true });
+    waku = await createLightNode({
+      staticNoiseKey: NOISE_KEY_1,
+      libp2p: { addresses: { listen: ["/ip4/0.0.0.0/tcp/0/ws"] } },
+    });
+    await waku.start();
+    await waku.dial(await nwaku.getMultiaddrWithId());
+    await waitForRemotePeer(waku);
+  });
+
+  it("Ephemeral messages are not stored", async function () {
+    this.timeout(15_000);
+
+    const asymText =
+      "This message is encrypted for me using asymmetric encryption";
+    const symText =
+      "This message is encrypted for me using symmetric encryption";
+    const clearText = "This is a clear text message";
+
+    const asymMsg = { payload: utf8ToBytes(asymText) };
+    const symMsg = {
+      payload: utf8ToBytes(symText),
+    };
+    const clearMsg = {
+      payload: utf8ToBytes(clearText),
+    };
+
+    const privateKey = generatePrivateKey();
+    const symKey = generateSymmetricKey();
+    const publicKey = getPublicKey(privateKey);
+
+    const AsymContentTopic = "/test/1/ephemeral-asym/utf8";
+    const SymContentTopic = "/test/1/ephemeral-sym/utf8";
+
+    const asymEncoder = new AsymEncoder(
+      AsymContentTopic,
+      publicKey,
+      undefined,
+      true
+    );
+    const symEncoder = new SymEncoder(SymContentTopic, symKey, undefined, true);
+    const clearEncoder = new EncoderV0(TestContentTopic, true);
+
+    const asymDecoder = new AsymDecoder(AsymContentTopic, privateKey);
+    const symDecoder = new SymDecoder(SymContentTopic, symKey);
+
+    const [waku1, waku2, nimWakuMultiaddr] = await Promise.all([
+      createFullNode({
+        staticNoiseKey: NOISE_KEY_1,
+      }).then((waku) => waku.start().then(() => waku)),
+      createFullNode({
+        staticNoiseKey: NOISE_KEY_2,
+      }).then((waku) => waku.start().then(() => waku)),
+      nwaku.getMultiaddrWithId(),
+    ]);
+
+    log("Waku nodes created");
+
+    await Promise.all([
+      waku1.dial(nimWakuMultiaddr),
+      waku2.dial(nimWakuMultiaddr),
+    ]);
+
+    log("Waku nodes connected to nwaku");
+
+    await waitForRemotePeer(waku1, [Protocols.LightPush]);
+
+    log("Sending messages using light push");
+    await Promise.all([
+      waku1.lightPush.push(asymEncoder, asymMsg),
+      waku1.lightPush.push(symEncoder, symMsg),
+      waku1.lightPush.push(clearEncoder, clearMsg),
+    ]);
+
+    await waitForRemotePeer(waku2, [Protocols.Store]);
+
+    const messages: DecodedMessage[] = [];
+    log("Retrieve messages from store");
+
+    for await (const msgPromises of waku2.store.queryGenerator([
+      asymDecoder,
+      symDecoder,
+      TestDecoder,
+    ])) {
+      for (const promise of msgPromises) {
+        const msg = await promise;
+        if (msg) {
+          messages.push(msg);
+        }
+      }
+    }
+
+    expect(messages?.length).eq(0);
+
+    !!waku1 && waku1.stop().catch((e) => console.log("Waku failed to stop", e));
+    !!waku2 && waku2.stop().catch((e) => console.log("Waku failed to stop", e));
+  });
+
+  it("Ephemeral field is preserved - encoder v0", async function () {
+    this.timeout(10000);
+
+    const ephemeralEncoder = new EncoderV0(TestContentTopic, true);
+
+    const messages: DecodedMessage[] = [];
+    const callback = (msg: DecodedMessage): void => {
+      messages.push(msg);
+    };
+    await waku.filter.subscribe([TestDecoder], callback);
+
+    await delay(200);
+    const normalTxt = "Normal message";
+    const ephemeralTxt = "Ephemeral Message";
+    await waku.lightPush.push(TestEncoder, {
+      payload: utf8ToBytes(normalTxt),
+    });
+    await waku.lightPush.push(ephemeralEncoder, {
+      payload: utf8ToBytes(ephemeralTxt),
+    });
+    while (messages.length < 2) {
+      await delay(250);
+    }
+
+    const normalMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === normalTxt
+    );
+    const ephemeralMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === ephemeralTxt
+    );
+
+    expect(normalMsg).to.not.be.undefined;
+    expect(ephemeralMsg).to.not.be.undefined;
+
+    expect(normalMsg!.ephemeral).to.be.false;
+    expect(ephemeralMsg!.ephemeral).to.be.true;
+  });
+
+  it("Ephemeral field is preserved - symmetric encryption", async function () {
+    this.timeout(10000);
+
+    const symKey = generateSymmetricKey();
+
+    const ephemeralEncoder = new SymEncoder(
+      TestContentTopic,
+      symKey,
+      undefined,
+      true
+    );
+    const encoder = new SymEncoder(TestContentTopic, symKey);
+    const decoder = new SymDecoder(TestContentTopic, symKey);
+
+    const messages: DecodedMessage[] = [];
+    const callback = (msg: DecodedMessage): void => {
+      messages.push(msg);
+    };
+    await waku.filter.subscribe([decoder], callback);
+
+    await delay(200);
+    const normalTxt = "Normal message";
+    const ephemeralTxt = "Ephemeral Message";
+    await waku.lightPush.push(encoder, {
+      payload: utf8ToBytes(normalTxt),
+    });
+    await waku.lightPush.push(ephemeralEncoder, {
+      payload: utf8ToBytes(ephemeralTxt),
+    });
+    while (messages.length < 2) {
+      await delay(250);
+    }
+
+    const normalMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === normalTxt
+    );
+    const ephemeralMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === ephemeralTxt
+    );
+
+    expect(normalMsg).to.not.be.undefined;
+    expect(ephemeralMsg).to.not.be.undefined;
+
+    expect(normalMsg!.ephemeral).to.be.false;
+    expect(ephemeralMsg!.ephemeral).to.be.true;
+  });
+
+  it("Ephemeral field is preserved - asymmetric encryption", async function () {
+    this.timeout(10000);
+
+    const privKey = generatePrivateKey();
+    const pubKey = getPublicKey(privKey);
+
+    const ephemeralEncoder = new AsymEncoder(
+      TestContentTopic,
+      pubKey,
+      undefined,
+      true
+    );
+    const encoder = new AsymEncoder(TestContentTopic, pubKey);
+    const decoder = new AsymDecoder(TestContentTopic, privKey);
+
+    const messages: DecodedMessage[] = [];
+    const callback = (msg: DecodedMessage): void => {
+      messages.push(msg);
+    };
+    await waku.filter.subscribe([decoder], callback);
+
+    await delay(200);
+    const normalTxt = "Normal message";
+    const ephemeralTxt = "Ephemeral Message";
+    await waku.lightPush.push(encoder, {
+      payload: utf8ToBytes(normalTxt),
+    });
+    await waku.lightPush.push(ephemeralEncoder, {
+      payload: utf8ToBytes(ephemeralTxt),
+    });
+    while (messages.length < 2) {
+      await delay(250);
+    }
+
+    const normalMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === normalTxt
+    );
+    const ephemeralMsg = messages.find(
+      (msg) => bytesToUtf8(msg.payload!) === ephemeralTxt
+    );
+
+    expect(normalMsg).to.not.be.undefined;
+    expect(ephemeralMsg).to.not.be.undefined;
+
+    expect(normalMsg!.ephemeral).to.be.false;
+    expect(ephemeralMsg!.ephemeral).to.be.true;
+  });
+});

packages/tests/tests/store.node.spec.ts

@@ -372,143 +372,6 @@ describe("Waku Store", () => {
     !!waku2 && waku2.stop().catch((e) => console.log("Waku failed to stop", e));
   });
 
-  it.skip("Ephemeral support", async function () {
-    this.timeout(15_000);
-
-    const asymText = "This message is encrypted for me using asymmetric";
-    const asymTopic = "/test/1/asymmetric/proto";
-
-    const symText =
-      "This message is encrypted for me using symmetric encryption";
-    const symTopic = "/test/1/symmetric/proto";
-
-    const clearText = "This is a clear text message for everyone to read";
-
-    const storeReadableText = "This message is readable by the store";
-    const storeUnreadableText = "This message is not readable by the store";
-
-    const timestamp = new Date();
-
-    const asymMsg = { payload: utf8ToBytes(asymText), timestamp };
-    const symMsg = {
-      payload: utf8ToBytes(symText),
-      timestamp: new Date(timestamp.valueOf() + 1),
-    };
-    const clearMsg = {
-      payload: utf8ToBytes(clearText),
-      timestamp: new Date(timestamp.valueOf() + 2),
-    };
-
-    const storeReadableMsg = {
-      payload: utf8ToBytes(storeReadableText),
-    };
-    const storeUnreadableMsg = {
-      payload: utf8ToBytes(storeUnreadableText),
-    };
-
-    const privateKey = generatePrivateKey();
-    const symKey = generateSymmetricKey();
-    const publicKey = getPublicKey(privateKey);
-
-    const storeWithAsymEncoder = new AsymEncoder(
-      asymTopic,
-      publicKey,
-      undefined,
-      false
-    );
-    const storeWithSymEncoder = new SymEncoder(
-      symTopic,
-      symKey,
-      undefined,
-      false
-    );
-
-    const dontStoreWithAsymEncoder = new AsymEncoder(
-      asymTopic,
-      publicKey,
-      undefined,
-      true
-    );
-    const dontStoreWithSymEncoder = new SymEncoder(
-      symTopic,
-      symKey,
-      undefined,
-      true
-    );
-
-    const storeEncoder = new EncoderV0(TestContentTopic, false);
-    const storeUnreadableEncoder = new EncoderV0(TestContentTopic, true);
-
-    const asymDecoder = new AsymDecoder(asymTopic, privateKey);
-    const symDecoder = new SymDecoder(symTopic, symKey);
-
-    const [waku1, waku2, nimWakuMultiaddr] = await Promise.all([
-      createFullNode({
-        staticNoiseKey: NOISE_KEY_1,
-      }).then((waku) => waku.start().then(() => waku)),
-      createFullNode({
-        staticNoiseKey: NOISE_KEY_2,
-      }).then((waku) => waku.start().then(() => waku)),
-      nwaku.getMultiaddrWithId(),
-    ]);
-
-    log("Waku nodes created");
-
-    await Promise.all([
-      waku1.dial(nimWakuMultiaddr),
-      waku2.dial(nimWakuMultiaddr),
-    ]);
-
-    log("Waku nodes connected to nwaku");
-
-    await waitForRemotePeer(waku1, [Protocols.LightPush]);
-
-    log("Sending messages using light push");
-    await Promise.all([
-      waku1.lightPush.push(storeWithAsymEncoder, asymMsg),
-      waku1.lightPush.push(storeWithSymEncoder, symMsg),
-      waku1.lightPush.push(dontStoreWithAsymEncoder, asymMsg),
-      waku1.lightPush.push(dontStoreWithSymEncoder, symMsg),
-      waku1.lightPush.push(TestEncoder, clearMsg),
-      waku1.lightPush.push(storeEncoder, storeReadableMsg),
-      waku1.lightPush.push(storeUnreadableEncoder, storeUnreadableMsg),
-    ]);
-
-    await waitForRemotePeer(waku2, [Protocols.Store]);
-
-    const messages: DecodedMessage[] = [];
-    log("Retrieve messages from store");
-
-    for await (const msgPromises of waku2.store.queryGenerator([
-      asymDecoder,
-      symDecoder,
-      TestDecoder,
-    ])) {
-      for (const promise of msgPromises) {
-        const msg = await promise;
-        if (msg) {
-          messages.push(msg);
-        }
-      }
-    }
-
-    // Messages are ordered from oldest to latest within a page (1 page query)
-    expect(bytesToUtf8(messages[0].payload!)).to.eq(asymText);
-    expect(bytesToUtf8(messages[1].payload!)).to.eq(symText);
-    expect(bytesToUtf8(messages[2].payload!)).to.eq(clearText);
-    expect(bytesToUtf8(messages[3].payload!)).to.eq(storeReadableText);
-    expect(messages?.length).eq(4);
-
-    // check for ephemeral
-    expect(messages[0].ephemeral).to.be.false;
-    expect(messages[1].ephemeral).to.be.false;
-    expect(messages[2].ephemeral).to.be.false;
-    expect(messages[3].ephemeral).to.be.false;
-
-    !!waku1 && waku1.stop().catch((e) => console.log("Waku failed to stop", e));
-    !!waku2 && waku2.stop().catch((e) => console.log("Waku failed to stop", e));
-  });
-
   it("Ordered callback, using start and end time", async function () {
     this.timeout(20000);