From bd983ea48ee73fda5a7137d5ef681965aeabb4a5 Mon Sep 17 00:00:00 2001 From: "fryorcraken.eth" Date: Fri, 10 Mar 2023 14:41:07 +1100 Subject: [PATCH] feat!: enable encoding of `meta` field --- .../src/lib/message/topic_only_message.ts | 2 + .../core/src/lib/message/version_0.spec.ts | 48 +++++++++++++++++++ packages/core/src/lib/message/version_0.ts | 27 +++++++++-- packages/core/src/lib/to_proto_message.ts | 1 + packages/interfaces/src/message.ts | 11 +++++ packages/message-encryption/src/ecies.spec.ts | 48 +++++++++++++++++++ packages/message-encryption/src/ecies.ts | 23 +++++++-- .../message-encryption/src/symmetric.spec.ts | 47 ++++++++++++++++++ packages/message-encryption/src/symmetric.ts | 18 +++++-- packages/proto/src/lib/filter.ts | 9 ++++ packages/proto/src/lib/light_push.ts | 9 ++++ packages/proto/src/lib/message.proto | 1 + packages/proto/src/lib/message.ts | 9 ++++ packages/proto/src/lib/store.ts | 9 ++++ 14 files changed, 252 insertions(+), 10 deletions(-) diff --git a/packages/core/src/lib/message/topic_only_message.ts b/packages/core/src/lib/message/topic_only_message.ts index 6bf90b3db4..fee30680aa 100644 --- a/packages/core/src/lib/message/topic_only_message.ts +++ b/packages/core/src/lib/message/topic_only_message.ts @@ -12,6 +12,7 @@ export class TopicOnlyMessage implements IDecodedMessage { public payload: Uint8Array = new Uint8Array(); public rateLimitProof: undefined; public timestamp: undefined; + public meta: undefined; public ephemeral: undefined; constructor( @@ -35,6 +36,7 @@ export class TopicOnlyDecoder implements IDecoder { payload: new Uint8Array(), rateLimitProof: undefined, timestamp: undefined, + meta: undefined, version: undefined, ephemeral: undefined, }); diff --git a/packages/core/src/lib/message/version_0.spec.ts b/packages/core/src/lib/message/version_0.spec.ts index 14a5e5bc08..e1158fe0a9 100644 --- a/packages/core/src/lib/message/version_0.spec.ts +++ b/packages/core/src/lib/message/version_0.spec.ts @@ -1,3 +1,4 @@ +import type { IProtoMessage } from "@waku/interfaces"; import { expect } from "chai"; import fc from "fast-check"; @@ -57,4 +58,51 @@ describe("Waku Message version 0", function () { ) ); }); + + it("Meta field set when metaSetter is specified", async function () { + await fc.assert( + fc.asyncProperty( + fc.string(), + fc.string(), + fc.uint8Array({ minLength: 1 }), + async (contentTopic, pubSubTopic, payload) => { + // Encode the length of the payload + // Not a relevant real life example + const metaSetter = ( + msg: IProtoMessage & { meta: undefined } + ): Uint8Array => { + const buffer = new ArrayBuffer(4); + const view = new DataView(buffer); + view.setUint32(0, msg.payload.length, false); + return new Uint8Array(buffer); + }; + + const encoder = createEncoder({ + contentTopic, + ephemeral: true, + metaSetter, + }); + const bytes = await encoder.toWire({ payload }); + const decoder = createDecoder(contentTopic); + const protoResult = await decoder.fromWireToProtoObj(bytes); + const result = (await decoder.fromProtoObj( + pubSubTopic, + protoResult! + )) as DecodedMessage; + + const expectedMeta = metaSetter({ + payload, + timestamp: undefined, + contentTopic: "", + ephemeral: undefined, + meta: undefined, + rateLimitProof: undefined, + version: undefined, + }); + + expect(result.meta).to.deep.eq(expectedMeta); + } + ) + ); + }); }); diff --git a/packages/core/src/lib/message/version_0.ts b/packages/core/src/lib/message/version_0.ts index 38bd22fa26..0a5376eec0 100644 --- a/packages/core/src/lib/message/version_0.ts +++ b/packages/core/src/lib/message/version_0.ts @@ -1,3 +1,4 @@ +import { IMetaSetter } from "@waku/interfaces"; import type { EncoderOptions, IDecodedMessage, @@ -50,6 +51,10 @@ export class DecodedMessage implements IDecodedMessage { } } + get meta(): Uint8Array | undefined { + return this.proto.meta; + } + get version(): number { // https://rfc.vac.dev/spec/14/ // > If omitted, the value SHOULD be interpreted as version 0. @@ -62,7 +67,11 @@ export class DecodedMessage implements IDecodedMessage { } export class Encoder implements IEncoder { - constructor(public contentTopic: string, public ephemeral: boolean = false) {} + constructor( + public contentTopic: string, + public ephemeral: boolean = false, + public metaSetter?: IMetaSetter + ) {} async toWire(message: IMessage): Promise { return proto.WakuMessage.encode(await this.toProtoObj(message)); @@ -71,14 +80,22 @@ export class Encoder implements IEncoder { async toProtoObj(message: IMessage): Promise { const timestamp = message.timestamp ?? new Date(); - return { + const protoMessage = { payload: message.payload, version: Version, contentTopic: this.contentTopic, timestamp: BigInt(timestamp.valueOf()) * OneMillion, + meta: undefined, rateLimitProof: message.rateLimitProof, ephemeral: this.ephemeral, }; + + if (this.metaSetter) { + const meta = this.metaSetter(protoMessage); + return { ...protoMessage, meta }; + } + + return protoMessage; } } @@ -94,8 +111,9 @@ export class Encoder implements IEncoder { export function createEncoder({ contentTopic, ephemeral, + metaSetter, }: EncoderOptions): Encoder { - return new Encoder(contentTopic, ephemeral); + return new Encoder(contentTopic, ephemeral, metaSetter); } export class Decoder implements IDecoder { @@ -109,6 +127,7 @@ export class Decoder implements IDecoder { contentTopic: protoMessage.contentTopic, version: protoMessage.version ?? undefined, timestamp: protoMessage.timestamp ?? undefined, + meta: protoMessage.meta ?? undefined, rateLimitProof: protoMessage.rateLimitProof ?? undefined, ephemeral: protoMessage.ephemeral ?? false, }); @@ -135,7 +154,7 @@ export class Decoder implements IDecoder { } /** - * Creates an decoder that decode messages without Waku level encryption. + * Creates a decoder that decode messages without Waku level encryption. * * A decoder is used to decode messages from the [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/) * format when received from the Waku network. The resulting decoder can then be diff --git a/packages/core/src/lib/to_proto_message.ts b/packages/core/src/lib/to_proto_message.ts index c9d968a1b9..102d14c409 100644 --- a/packages/core/src/lib/to_proto_message.ts +++ b/packages/core/src/lib/to_proto_message.ts @@ -6,6 +6,7 @@ const EmptyMessage: IProtoMessage = { contentTopic: "", version: undefined, timestamp: undefined, + meta: undefined, rateLimitProof: undefined, ephemeral: undefined, }; diff --git a/packages/interfaces/src/message.ts b/packages/interfaces/src/message.ts index d2e52f04f0..c16d700109 100644 --- a/packages/interfaces/src/message.ts +++ b/packages/interfaces/src/message.ts @@ -17,6 +17,7 @@ export interface IProtoMessage { contentTopic: string; version: number | undefined; timestamp: bigint | undefined; + meta: Uint8Array | undefined; rateLimitProof: IRateLimitProof | undefined; ephemeral: boolean | undefined; } @@ -30,6 +31,10 @@ export interface IMessage { rateLimitProof?: IRateLimitProof; } +export interface IMetaSetter { + (message: IProtoMessage & { meta: undefined }): Uint8Array; +} + export interface EncoderOptions { /** The content topic to set on outgoing messages. */ contentTopic: string; @@ -38,6 +43,12 @@ export interface EncoderOptions { * @defaultValue `false` */ ephemeral?: boolean; + /** + * A function called when encoding messages to set the meta field. + * @param IProtoMessage The message encoded for wire, without the meta field. + * If encryption is used, `metaSetter` only accesses _encrypted_ payload. + */ + metaSetter?: IMetaSetter; } export interface IEncoder { diff --git a/packages/message-encryption/src/ecies.spec.ts b/packages/message-encryption/src/ecies.spec.ts index 71d68dc87b..698ecddeb0 100644 --- a/packages/message-encryption/src/ecies.spec.ts +++ b/packages/message-encryption/src/ecies.spec.ts @@ -1,3 +1,4 @@ +import { IProtoMessage } from "@waku/interfaces"; import { expect } from "chai"; import fc from "fast-check"; @@ -81,4 +82,51 @@ describe("Ecies Encryption", function () { ) ); }); + + it("Check meta is set [ecies]", async function () { + await fc.assert( + fc.asyncProperty( + fc.string(), + fc.string(), + fc.uint8Array({ minLength: 1 }), + fc.uint8Array({ min: 1, minLength: 32, maxLength: 32 }), + async (pubSubTopic, contentTopic, payload, privateKey) => { + const publicKey = getPublicKey(privateKey); + const metaSetter = ( + msg: IProtoMessage & { meta: undefined } + ): Uint8Array => { + const buffer = new ArrayBuffer(4); + const view = new DataView(buffer); + view.setUint32(0, msg.payload.length, false); + return new Uint8Array(buffer); + }; + + const encoder = createEncoder({ + contentTopic, + publicKey, + metaSetter, + }); + const bytes = await encoder.toWire({ payload }); + + const decoder = createDecoder(contentTopic, privateKey); + const protoResult = await decoder.fromWireToProtoObj(bytes!); + if (!protoResult) throw "Failed to proto decode"; + const result = await decoder.fromProtoObj(pubSubTopic, protoResult); + if (!result) throw "Failed to decode"; + + const expectedMeta = metaSetter({ + payload: protoResult.payload, + timestamp: undefined, + contentTopic: "", + ephemeral: undefined, + meta: undefined, + rateLimitProof: undefined, + version: undefined, + }); + + expect(result.meta).to.deep.equal(expectedMeta); + } + ) + ); + }); }); diff --git a/packages/message-encryption/src/ecies.ts b/packages/message-encryption/src/ecies.ts index c991db9c69..98d241e6e2 100644 --- a/packages/message-encryption/src/ecies.ts +++ b/packages/message-encryption/src/ecies.ts @@ -1,4 +1,5 @@ import { Decoder as DecoderV0 } from "@waku/core/lib/message/version_0"; +import { IMetaSetter } from "@waku/interfaces"; import type { EncoderOptions as BaseEncoderOptions, IDecoder, @@ -34,7 +35,8 @@ class Encoder implements IEncoder { public contentTopic: string, private publicKey: Uint8Array, private sigPrivKey?: Uint8Array, - public ephemeral: boolean = false + public ephemeral: boolean = false, + public metaSetter?: IMetaSetter ) {} async toWire(message: IMessage): Promise { @@ -50,14 +52,22 @@ class Encoder implements IEncoder { const payload = await encryptAsymmetric(preparedPayload, this.publicKey); - return { + const protoMessage = { payload, version: Version, contentTopic: this.contentTopic, timestamp: BigInt(timestamp.valueOf()) * OneMillion, + meta: undefined, rateLimitProof: message.rateLimitProof, ephemeral: this.ephemeral, }; + + if (this.metaSetter) { + const meta = this.metaSetter(protoMessage); + return { ...protoMessage, meta }; + } + + return protoMessage; } } @@ -85,8 +95,15 @@ export function createEncoder({ publicKey, sigPrivKey, ephemeral = false, + metaSetter, }: EncoderOptions): Encoder { - return new Encoder(contentTopic, publicKey, sigPrivKey, ephemeral); + return new Encoder( + contentTopic, + publicKey, + sigPrivKey, + ephemeral, + metaSetter + ); } class Decoder extends DecoderV0 implements IDecoder { diff --git a/packages/message-encryption/src/symmetric.spec.ts b/packages/message-encryption/src/symmetric.spec.ts index 837c88fa10..cf724d2fa0 100644 --- a/packages/message-encryption/src/symmetric.spec.ts +++ b/packages/message-encryption/src/symmetric.spec.ts @@ -1,3 +1,4 @@ +import { IProtoMessage } from "@waku/interfaces"; import { expect } from "chai"; import fc from "fast-check"; @@ -70,4 +71,50 @@ describe("Symmetric Encryption", function () { ) ); }); + + it("Check meta is set [symmetric]", async function () { + await fc.assert( + fc.asyncProperty( + fc.string(), + fc.string(), + fc.uint8Array({ minLength: 1 }), + fc.uint8Array({ min: 1, minLength: 32, maxLength: 32 }), + async (pubSubTopic, contentTopic, payload, symKey) => { + const metaSetter = ( + msg: IProtoMessage & { meta: undefined } + ): Uint8Array => { + const buffer = new ArrayBuffer(4); + const view = new DataView(buffer); + view.setUint32(0, msg.payload.length, false); + return new Uint8Array(buffer); + }; + + const encoder = createEncoder({ + contentTopic, + symKey, + metaSetter, + }); + const bytes = await encoder.toWire({ payload }); + + const decoder = createDecoder(contentTopic, symKey); + const protoResult = await decoder.fromWireToProtoObj(bytes!); + if (!protoResult) throw "Failed to proto decode"; + const result = await decoder.fromProtoObj(pubSubTopic, protoResult); + if (!result) throw "Failed to decode"; + + const expectedMeta = metaSetter({ + payload: protoResult.payload, + timestamp: undefined, + contentTopic: "", + ephemeral: undefined, + meta: undefined, + rateLimitProof: undefined, + version: undefined, + }); + + expect(result.meta).to.deep.equal(expectedMeta); + } + ) + ); + }); }); diff --git a/packages/message-encryption/src/symmetric.ts b/packages/message-encryption/src/symmetric.ts index 6e18301b7f..53bf05ee38 100644 --- a/packages/message-encryption/src/symmetric.ts +++ b/packages/message-encryption/src/symmetric.ts @@ -4,6 +4,7 @@ import type { IDecoder, IEncoder, IMessage, + IMetaSetter, IProtoMessage, } from "@waku/interfaces"; import { WakuMessage } from "@waku/proto"; @@ -29,7 +30,8 @@ class Encoder implements IEncoder { public contentTopic: string, private symKey: Uint8Array, private sigPrivKey?: Uint8Array, - public ephemeral: boolean = false + public ephemeral: boolean = false, + public metaSetter?: IMetaSetter ) {} async toWire(message: IMessage): Promise { @@ -44,14 +46,23 @@ class Encoder implements IEncoder { const preparedPayload = await preCipher(message.payload, this.sigPrivKey); const payload = await encryptSymmetric(preparedPayload, this.symKey); - return { + + const protoMessage = { payload, version: Version, contentTopic: this.contentTopic, timestamp: BigInt(timestamp.valueOf()) * OneMillion, + meta: undefined, rateLimitProof: message.rateLimitProof, ephemeral: this.ephemeral, }; + + if (this.metaSetter) { + const meta = this.metaSetter(protoMessage); + return { ...protoMessage, meta }; + } + + return protoMessage; } } @@ -80,8 +91,9 @@ export function createEncoder({ symKey, sigPrivKey, ephemeral = false, + metaSetter, }: EncoderOptions): Encoder { - return new Encoder(contentTopic, symKey, sigPrivKey, ephemeral); + return new Encoder(contentTopic, symKey, sigPrivKey, ephemeral, metaSetter); } class Decoder extends DecoderV0 implements IDecoder { diff --git a/packages/proto/src/lib/filter.ts b/packages/proto/src/lib/filter.ts index 13659c7d8e..1445132018 100644 --- a/packages/proto/src/lib/filter.ts +++ b/packages/proto/src/lib/filter.ts @@ -430,6 +430,7 @@ export interface WakuMessage { contentTopic: string; version?: number; timestamp?: bigint; + meta?: Uint8Array; rateLimitProof?: RateLimitProof; ephemeral?: boolean; } @@ -465,6 +466,11 @@ export namespace WakuMessage { w.sint64(obj.timestamp); } + if (obj.meta != null) { + w.uint32(90); + w.bytes(obj.meta); + } + if (obj.rateLimitProof != null) { w.uint32(170); RateLimitProof.codec().encode(obj.rateLimitProof, w); @@ -503,6 +509,9 @@ export namespace WakuMessage { case 10: obj.timestamp = reader.sint64(); break; + case 11: + obj.meta = reader.bytes(); + break; case 21: obj.rateLimitProof = RateLimitProof.codec().decode( reader, diff --git a/packages/proto/src/lib/light_push.ts b/packages/proto/src/lib/light_push.ts index de186f2603..4a1f609b50 100644 --- a/packages/proto/src/lib/light_push.ts +++ b/packages/proto/src/lib/light_push.ts @@ -362,6 +362,7 @@ export interface WakuMessage { contentTopic: string; version?: number; timestamp?: bigint; + meta?: Uint8Array; rateLimitProof?: RateLimitProof; ephemeral?: boolean; } @@ -397,6 +398,11 @@ export namespace WakuMessage { w.sint64(obj.timestamp); } + if (obj.meta != null) { + w.uint32(90); + w.bytes(obj.meta); + } + if (obj.rateLimitProof != null) { w.uint32(170); RateLimitProof.codec().encode(obj.rateLimitProof, w); @@ -435,6 +441,9 @@ export namespace WakuMessage { case 10: obj.timestamp = reader.sint64(); break; + case 11: + obj.meta = reader.bytes(); + break; case 21: obj.rateLimitProof = RateLimitProof.codec().decode( reader, diff --git a/packages/proto/src/lib/message.proto b/packages/proto/src/lib/message.proto index 93c573f4d1..dcb9bac151 100644 --- a/packages/proto/src/lib/message.proto +++ b/packages/proto/src/lib/message.proto @@ -17,6 +17,7 @@ message WakuMessage { string content_topic = 2; optional uint32 version = 3; optional sint64 timestamp = 10; + optional bytes meta = 11; optional RateLimitProof rate_limit_proof = 21; optional bool ephemeral = 31; } diff --git a/packages/proto/src/lib/message.ts b/packages/proto/src/lib/message.ts index a724b7222a..db22724f26 100644 --- a/packages/proto/src/lib/message.ts +++ b/packages/proto/src/lib/message.ts @@ -134,6 +134,7 @@ export interface WakuMessage { contentTopic: string; version?: number; timestamp?: bigint; + meta?: Uint8Array; rateLimitProof?: RateLimitProof; ephemeral?: boolean; } @@ -169,6 +170,11 @@ export namespace WakuMessage { w.sint64(obj.timestamp); } + if (obj.meta != null) { + w.uint32(90); + w.bytes(obj.meta); + } + if (obj.rateLimitProof != null) { w.uint32(170); RateLimitProof.codec().encode(obj.rateLimitProof, w); @@ -207,6 +213,9 @@ export namespace WakuMessage { case 10: obj.timestamp = reader.sint64(); break; + case 11: + obj.meta = reader.bytes(); + break; case 21: obj.rateLimitProof = RateLimitProof.codec().decode( reader, diff --git a/packages/proto/src/lib/store.ts b/packages/proto/src/lib/store.ts index 6d9a663501..881d6d2d58 100644 --- a/packages/proto/src/lib/store.ts +++ b/packages/proto/src/lib/store.ts @@ -676,6 +676,7 @@ export interface WakuMessage { contentTopic: string; version?: number; timestamp?: bigint; + meta?: Uint8Array; rateLimitProof?: RateLimitProof; ephemeral?: boolean; } @@ -711,6 +712,11 @@ export namespace WakuMessage { w.sint64(obj.timestamp); } + if (obj.meta != null) { + w.uint32(90); + w.bytes(obj.meta); + } + if (obj.rateLimitProof != null) { w.uint32(170); RateLimitProof.codec().encode(obj.rateLimitProof, w); @@ -749,6 +755,9 @@ export namespace WakuMessage { case 10: obj.timestamp = reader.sint64(); break; + case 11: + obj.meta = reader.bytes(); + break; case 21: obj.rateLimitProof = RateLimitProof.codec().decode( reader,