diff --git a/package-lock.json b/package-lock.json index 721552f851..e8bb2d1cce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26315,6 +26315,7 @@ "version": "0.0.13", "license": "MIT OR Apache-2.0", "dependencies": { + "@noble/hashes": "^1.3.2", "chai": "^4.3.8", "debug": "^4.3.4", "uint8arrays": "^4.0.4" @@ -29590,6 +29591,7 @@ "@waku/utils": { "version": "file:packages/utils", "requires": { + "@noble/hashes": "^1.3.2", "@rollup/plugin-commonjs": "^25.0.4", "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.3", diff --git a/packages/utils/package.json b/packages/utils/package.json index ed2c44adca..22370e2d11 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -67,6 +67,7 @@ "node": ">=18" }, "dependencies": { + "@noble/hashes": "^1.3.2", "chai": "^4.3.8", "debug": "^4.3.4", "uint8arrays": "^4.0.4" diff --git a/packages/utils/src/common/sharding.spec.ts b/packages/utils/src/common/sharding.spec.ts index f2963e6b09..fd206dd788 100644 --- a/packages/utils/src/common/sharding.spec.ts +++ b/packages/utils/src/common/sharding.spec.ts @@ -1,6 +1,6 @@ import { expect } from "chai"; -import { ensureValidContentTopic } from "./sharding"; +import { contentTopicToShardIndex, ensureValidContentTopic } from "./sharding"; const testInvalidCases = ( contentTopics: string[], @@ -86,3 +86,15 @@ describe("ensureValidContentTopic", () => { testInvalidCases(["/0/myapp/1/mytopic/"], "Encoding field cannot be empty"); }); }); + +describe("contentTopicToShardIndex", () => { + it("converts content topics to expected shard index", () => { + const contentTopics: [string, number][] = [ + ["/toychat/2/huilong/proto", 3], + ["/myapp/1/latest/proto", 0] + ]; + for (const [topic, shard] of contentTopics) { + expect(contentTopicToShardIndex(topic)).to.eq(shard); + } + }); +}); diff --git a/packages/utils/src/common/sharding.ts b/packages/utils/src/common/sharding.ts index 302229b4e8..e935ed11c6 100644 --- a/packages/utils/src/common/sharding.ts +++ b/packages/utils/src/common/sharding.ts @@ -1,5 +1,8 @@ +import { sha256 } from "@noble/hashes/sha256"; import type { PubsubTopic, ShardInfo } from "@waku/interfaces"; +import { concat, utf8ToBytes } from "../bytes/index.js"; + export const shardInfoToPubsubTopics = ( shardInfo: ShardInfo ): PubsubTopic[] => { @@ -19,18 +22,28 @@ export function ensurePubsubTopicIsConfigured( } } +interface ContentTopic { + generation: number; + application: string; + version: string; + topicName: string; + encoding: string; +} + /** * Given a string, will throw an error if it is not formatted as a valid content topic for autosharding based on https://rfc.vac.dev/spec/51/ * @param contentTopic String to validate + * @returns Object with each content topic field as an attribute */ -export function ensureValidContentTopic(contentTopic: string): void { +export function ensureValidContentTopic(contentTopic: string): ContentTopic { const parts = contentTopic.split("/"); if (parts.length < 5 || parts.length > 6) { throw Error("Content topic format is invalid"); } // Validate generation field if present + let generation = 0; if (parts.length == 6) { - const generation = parseInt(parts[1]); + generation = parseInt(parts[1]); if (isNaN(generation)) { throw new Error("Invalid generation field in content topic"); } @@ -56,4 +69,37 @@ export function ensureValidContentTopic(contentTopic: string): void { if (fields[3].length == 0) { throw new Error("Encoding field cannot be empty"); } + + return { + generation, + application: fields[0], + version: fields[1], + topicName: fields[2], + encoding: fields[3] + }; +} + +/** + * Given a string, determines which autoshard index to use for its pubsub topic. + * Based on the algorithm described in the RFC: https://rfc.vac.dev/spec/51//#algorithm + */ +export function contentTopicToShardIndex( + contentTopic: string, + networkShards: number = 8 +): number { + const { application, version } = ensureValidContentTopic(contentTopic); + const digest = sha256( + concat([utf8ToBytes(application), utf8ToBytes(version)]) + ); + const dataview = new DataView(digest.buffer.slice(-8)); + return Number(dataview.getBigUint64(0, false) % BigInt(networkShards)); +} + +export function contentTopicToPubsubTopic( + contentTopic: string, + clusterId: number = 1, + networkShards: number = 8 +): string { + const shardIndex = contentTopicToShardIndex(contentTopic, networkShards); + return `/waku/2/rs/${clusterId}/${shardIndex}`; }