mirror of
https://github.com/logos-messaging/js-waku.git
synced 2026-01-02 13:53:12 +00:00
Merge pull request #2233 from waku-org/feat/init-sds
feat(sds): create package for sds and add protobuf def
This commit is contained in:
Arseniy Klempner
GitHub
commit
6abd2d18a1
186
package-lock.json
generated
186
package-lock.json
generated
@ -19,7 +19,8 @@
|
||||
"packages/tests",
|
||||
"packages/browser-tests",
|
||||
"packages/build-utils",
|
||||
"packages/react-native-polyfills"
|
||||
"packages/react-native-polyfills",
|
||||
"packages/sds"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@size-limit/preset-big-lib": "^11.0.2",
|
||||
@ -10805,6 +10806,10 @@
|
||||
"resolved": "packages/sdk",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@waku/sds": {
|
||||
"resolved": "packages/sds",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@waku/tests": {
|
||||
"resolved": "packages/tests",
|
||||
"link": true
|
||||
@ -11684,64 +11689,6 @@
|
||||
"ajv": "^6.9.1"
|
||||
}
|
||||
},
|
||||
"node_modules/allure-commandline": {
|
||||
"version": "2.32.0",
|
||||
"resolved": "https://registry.npmjs.org/allure-commandline/-/allure-commandline-2.32.0.tgz",
|
||||
"integrity": "sha512-W03ors+ks8uy0SgQILHQvtvR0iadAfDYmTFC3p8Pk4pi8KXUW1cF+z8FN2+7deH3FE2cuYgjhhA+CdLdJfzOMQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"allure": "bin/allure"
|
||||
}
|
||||
},
|
||||
"node_modules/allure-js-commons": {
|
||||
"version": "2.15.1",
|
||||
"resolved": "https://registry.npmjs.org/allure-js-commons/-/allure-js-commons-2.15.1.tgz",
|
||||
"integrity": "sha512-5V/VINplbu0APnfSZOkYpKOzucO36Q2EtTD1kqjWjl7n6tj7Hh+IHCZsH3Vpk/LXRDfj9RuXugBBvwYKV5YMJw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"md5": "^2.3.0",
|
||||
"properties": "^1.2.1",
|
||||
"strip-ansi": "^5.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/allure-js-commons/node_modules/ansi-regex": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
|
||||
"integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/allure-js-commons/node_modules/strip-ansi": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
|
||||
"integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-regex": "^4.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/allure-mocha": {
|
||||
"version": "2.15.1",
|
||||
"resolved": "https://registry.npmjs.org/allure-mocha/-/allure-mocha-2.15.1.tgz",
|
||||
"integrity": "sha512-4Hk2qUR6LdAUXNpPe73MV3DPKrBH7zy57lbAdb/D0poNIkdGEkzUYkpVPtW1imYfjqFXKBFEPOSJWqznGuiyjg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"allure-js-commons": "2.15.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"mocha": ">=6.2.x"
|
||||
}
|
||||
},
|
||||
"node_modules/anser": {
|
||||
"version": "1.4.10",
|
||||
"resolved": "https://registry.npmjs.org/anser/-/anser-1.4.10.tgz",
|
||||
@ -25835,23 +25782,6 @@
|
||||
"node": ">= 14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mocha-multi-reporters": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/mocha-multi-reporters/-/mocha-multi-reporters-1.5.1.tgz",
|
||||
"integrity": "sha512-Yb4QJOaGLIcmB0VY7Wif5AjvLMUFAdV57D2TWEva1Y0kU/3LjKpeRVmlMIfuO1SVbauve459kgtIizADqxMWPg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.1.1",
|
||||
"lodash": "^4.17.15"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"mocha": ">=3.1.2"
|
||||
}
|
||||
},
|
||||
"node_modules/mocha/node_modules/cliui": {
|
||||
"version": "7.0.4",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
|
||||
@ -33016,16 +32946,6 @@
|
||||
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/properties": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/properties/-/properties-1.2.1.tgz",
|
||||
"integrity": "sha512-qYNxyMj1JeW54i/EWEFsM1cVwxJbtgPp8+0Wg9XjNaK6VE/c4oRi6PNu5p7w1mNXEIQIjV5Wwn8v8Gz82/QzdQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/proto-list": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
|
||||
@ -40715,6 +40635,24 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"packages/scalable-data-sync": {
|
||||
"version": "0.0.1",
|
||||
"extraneous": true,
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-commonjs": "^25.0.7",
|
||||
"@rollup/plugin-json": "^6.0.0",
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
"@waku/build-utils": "*",
|
||||
"cspell": "^8.6.1",
|
||||
"fast-check": "^3.19.0",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"rollup": "^4.12.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"packages/sdk": {
|
||||
"name": "@waku/sdk",
|
||||
"version": "0.0.29",
|
||||
@ -40841,6 +40779,79 @@
|
||||
"url": "https://opencollective.com/sinon"
|
||||
}
|
||||
},
|
||||
"packages/sds": {
|
||||
"name": "@waku/sds",
|
||||
"version": "0.0.1",
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"dependencies": {
|
||||
"chai": "^5.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-commonjs": "^25.0.7",
|
||||
"@rollup/plugin-json": "^6.0.0",
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
"@waku/build-utils": "*",
|
||||
"cspell": "^8.6.1",
|
||||
"fast-check": "^3.19.0",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"rollup": "^4.12.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"packages/sds/node_modules/assertion-error": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
|
||||
"integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"packages/sds/node_modules/chai": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz",
|
||||
"integrity": "sha512-aGtmf24DW6MLHHG5gCx4zaI3uBq3KRtxeVs0DjFH6Z0rDNbsvTxFASFvdj79pxjxZ8/5u3PIiN3IwEIQkiiuPw==",
|
||||
"dependencies": {
|
||||
"assertion-error": "^2.0.1",
|
||||
"check-error": "^2.1.1",
|
||||
"deep-eql": "^5.0.1",
|
||||
"loupe": "^3.1.0",
|
||||
"pathval": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"packages/sds/node_modules/check-error": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz",
|
||||
"integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==",
|
||||
"engines": {
|
||||
"node": ">= 16"
|
||||
}
|
||||
},
|
||||
"packages/sds/node_modules/deep-eql": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz",
|
||||
"integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"packages/sds/node_modules/loupe": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.3.tgz",
|
||||
"integrity": "sha512-kkIp7XSkP78ZxJEsSxW3712C6teJVoeHHwgo9zJ380de7IYyJ2ISlxojcH2pC5OFLewESmnRi/+XCDIEEVyoug=="
|
||||
},
|
||||
"packages/sds/node_modules/pathval": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.0.tgz",
|
||||
"integrity": "sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==",
|
||||
"engines": {
|
||||
"node": ">= 14.16"
|
||||
}
|
||||
},
|
||||
"packages/tests": {
|
||||
"name": "@waku/tests",
|
||||
"version": "0.0.1",
|
||||
@ -40875,8 +40886,6 @@
|
||||
"@waku/message-encryption": "*",
|
||||
"@waku/relay": "*",
|
||||
"@waku/sdk": "*",
|
||||
"allure-commandline": "^2.27.0",
|
||||
"allure-mocha": "^2.9.2",
|
||||
"chai": "^4.3.10",
|
||||
"cspell": "^8.6.1",
|
||||
"datastore-core": "^10.0.2",
|
||||
@ -40884,7 +40893,6 @@
|
||||
"interface-datastore": "^8.2.10",
|
||||
"libp2p": "2.1.8",
|
||||
"mocha": "^10.3.0",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"npm-run-all": "^4.1.5"
|
||||
},
|
||||
"engines": {
|
||||
|
||||
@ -16,7 +16,8 @@
|
||||
"packages/tests",
|
||||
"packages/browser-tests",
|
||||
"packages/build-utils",
|
||||
"packages/react-native-polyfills"
|
||||
"packages/react-native-polyfills",
|
||||
"packages/sds"
|
||||
],
|
||||
"scripts": {
|
||||
"prepare": "husky",
|
||||
|
||||
126
packages/proto/src/generated/sds_message.ts
Normal file
126
packages/proto/src/generated/sds_message.ts
Normal file
@ -0,0 +1,126 @@
|
||||
/* eslint-disable import/export */
|
||||
/* eslint-disable complexity */
|
||||
/* eslint-disable @typescript-eslint/no-namespace */
|
||||
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
|
||||
/* eslint-disable @typescript-eslint/no-empty-interface */
|
||||
|
||||
import { type Codec, decodeMessage, type DecodeOptions, encodeMessage, MaxLengthError, message } from 'protons-runtime'
|
||||
import type { Uint8ArrayList } from 'uint8arraylist'
|
||||
|
||||
export interface SdsMessage {
|
||||
messageId: string
|
||||
channelId: string
|
||||
lamportTimestamp?: number
|
||||
causalHistory: string[]
|
||||
bloomFilter?: Uint8Array
|
||||
content?: Uint8Array
|
||||
}
|
||||
|
||||
export namespace SdsMessage {
|
||||
let _codec: Codec<SdsMessage>
|
||||
|
||||
export const codec = (): Codec<SdsMessage> => {
|
||||
if (_codec == null) {
|
||||
_codec = message<SdsMessage>((obj, w, opts = {}) => {
|
||||
if (opts.lengthDelimited !== false) {
|
||||
w.fork()
|
||||
}
|
||||
|
||||
if ((obj.messageId != null && obj.messageId !== '')) {
|
||||
w.uint32(18)
|
||||
w.string(obj.messageId)
|
||||
}
|
||||
|
||||
if ((obj.channelId != null && obj.channelId !== '')) {
|
||||
w.uint32(26)
|
||||
w.string(obj.channelId)
|
||||
}
|
||||
|
||||
if (obj.lamportTimestamp != null) {
|
||||
w.uint32(80)
|
||||
w.int32(obj.lamportTimestamp)
|
||||
}
|
||||
|
||||
if (obj.causalHistory != null) {
|
||||
for (const value of obj.causalHistory) {
|
||||
w.uint32(90)
|
||||
w.string(value)
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.bloomFilter != null) {
|
||||
w.uint32(98)
|
||||
w.bytes(obj.bloomFilter)
|
||||
}
|
||||
|
||||
if (obj.content != null) {
|
||||
w.uint32(162)
|
||||
w.bytes(obj.content)
|
||||
}
|
||||
|
||||
if (opts.lengthDelimited !== false) {
|
||||
w.ldelim()
|
||||
}
|
||||
}, (reader, length, opts = {}) => {
|
||||
const obj: any = {
|
||||
messageId: '',
|
||||
channelId: '',
|
||||
causalHistory: []
|
||||
}
|
||||
|
||||
const end = length == null ? reader.len : reader.pos + length
|
||||
|
||||
while (reader.pos < end) {
|
||||
const tag = reader.uint32()
|
||||
|
||||
switch (tag >>> 3) {
|
||||
case 2: {
|
||||
obj.messageId = reader.string()
|
||||
break
|
||||
}
|
||||
case 3: {
|
||||
obj.channelId = reader.string()
|
||||
break
|
||||
}
|
||||
case 10: {
|
||||
obj.lamportTimestamp = reader.int32()
|
||||
break
|
||||
}
|
||||
case 11: {
|
||||
if (opts.limits?.causalHistory != null && obj.causalHistory.length === opts.limits.causalHistory) {
|
||||
throw new MaxLengthError('Decode error - map field "causalHistory" had too many elements')
|
||||
}
|
||||
|
||||
obj.causalHistory.push(reader.string())
|
||||
break
|
||||
}
|
||||
case 12: {
|
||||
obj.bloomFilter = reader.bytes()
|
||||
break
|
||||
}
|
||||
case 20: {
|
||||
obj.content = reader.bytes()
|
||||
break
|
||||
}
|
||||
default: {
|
||||
reader.skipType(tag & 7)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return obj
|
||||
})
|
||||
}
|
||||
|
||||
return _codec
|
||||
}
|
||||
|
||||
export const encode = (obj: Partial<SdsMessage>): Uint8Array => {
|
||||
return encodeMessage(obj, SdsMessage.codec())
|
||||
}
|
||||
|
||||
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<SdsMessage>): SdsMessage => {
|
||||
return decodeMessage(buf, SdsMessage.codec(), opts)
|
||||
}
|
||||
}
|
||||
11
packages/proto/src/lib/sds_message.proto
Normal file
11
packages/proto/src/lib/sds_message.proto
Normal file
@ -0,0 +1,11 @@
|
||||
syntax = "proto3";
|
||||
|
||||
message SdsMessage {
|
||||
// 1 Reserved for sender/participant id
|
||||
string message_id = 2; // Unique identifier of the message
|
||||
string channel_id = 3; // Identifier of the channel to which the message belongs
|
||||
optional int32 lamport_timestamp = 10; // Logical timestamp for causal ordering in channel
|
||||
repeated string causal_history = 11; // List of preceding message IDs that this message causally depends on. Generally 2 or 3 message IDs are included.
|
||||
optional bytes bloom_filter = 12; // Bloom filter representing received message IDs in channel
|
||||
optional bytes content = 20; // Actual content of the message
|
||||
}
|
||||
6
packages/sds/.eslintrc.cjs
Normal file
6
packages/sds/.eslintrc.cjs
Normal file
@ -0,0 +1,6 @@
|
||||
module.exports = {
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
project: "./tsconfig.dev.json",
|
||||
},
|
||||
};
|
||||
27
packages/sds/.mocharc.cjs
Normal file
27
packages/sds/.mocharc.cjs
Normal file
@ -0,0 +1,27 @@
|
||||
const config = {
|
||||
extension: ['ts'],
|
||||
spec: 'src/**/*.spec.ts',
|
||||
require: ['ts-node/register', 'isomorphic-fetch'],
|
||||
loader: 'ts-node/esm',
|
||||
'node-option': [
|
||||
'experimental-specifier-resolution=node',
|
||||
'loader=ts-node/esm'
|
||||
],
|
||||
exit: true,
|
||||
retries: 4
|
||||
};
|
||||
|
||||
if (process.env.CI) {
|
||||
console.log("Running tests in parallel");
|
||||
config.parallel = true;
|
||||
config.jobs = 6;
|
||||
console.log("Using JSON reporter for test results");
|
||||
config.reporter = 'json';
|
||||
config.reporterOptions = {
|
||||
output: 'reports/mocha-results.json'
|
||||
};
|
||||
} else {
|
||||
console.log("Running tests serially. To enable parallel execution update mocha config");
|
||||
}
|
||||
|
||||
module.exports = config;
|
||||
3
packages/sds/README.md
Normal file
3
packages/sds/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
# Scalable Data Sync
|
||||
|
||||
Typescript implementation of the [Scalable Data Sync protocol](https://github.com/vacp2p/rfc-index/blob/main/vac/raw/sds.md) for message reliability of distributed logs in the browser.
|
||||
84
packages/sds/package.json
Normal file
84
packages/sds/package.json
Normal file
@ -0,0 +1,84 @@
|
||||
{
|
||||
"name": "@waku/sds",
|
||||
"version": "0.0.1",
|
||||
"description": "Scalable Data Sync implementation for the browser. Based on https://github.com/vacp2p/rfc-index/blob/main/vac/raw/sds.md",
|
||||
"types": "./dist/index.d.ts",
|
||||
"module": "./dist/index.js",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"typesVersions": {
|
||||
"*": {
|
||||
"*": [
|
||||
"*",
|
||||
"dist/*",
|
||||
"dist/*/index"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "module",
|
||||
"author": "Waku Team",
|
||||
"homepage": "https://github.com/waku-org/js-waku/tree/master/packages/scalable-data-sync#readme",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/waku-org/js-waku.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/waku-org/js-waku/issues"
|
||||
},
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"keywords": [
|
||||
"waku",
|
||||
"decentralized",
|
||||
"secure",
|
||||
"communication",
|
||||
"web3",
|
||||
"ethereum",
|
||||
"dapps",
|
||||
"privacy"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "run-s build:**",
|
||||
"build:esm": "tsc",
|
||||
"build:bundle": "rollup --config rollup.config.js",
|
||||
"fix": "run-s fix:*",
|
||||
"fix:lint": "eslint src *.js --fix",
|
||||
"check": "run-s check:*",
|
||||
"check:lint": "eslint src *.js",
|
||||
"check:spelling": "cspell \"{README.md,src/**/*.ts}\"",
|
||||
"check:tsc": "tsc -p tsconfig.dev.json",
|
||||
"prepublish": "npm run build",
|
||||
"reset-hard": "git clean -dfx -e .idea && git reset --hard && npm i && npm run build",
|
||||
"test": "NODE_ENV=test run-s test:*",
|
||||
"test:node": "NODE_ENV=test TS_NODE_PROJECT=./tsconfig.dev.json mocha"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"dependencies": {
|
||||
"chai": "^5.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-commonjs": "^25.0.7",
|
||||
"@rollup/plugin-json": "^6.0.0",
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
"@waku/build-utils": "*",
|
||||
"cspell": "^8.6.1",
|
||||
"fast-check": "^3.19.0",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"rollup": "^4.12.0"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"bundle",
|
||||
"src/**/*.ts",
|
||||
"!**/*.spec.*",
|
||||
"!**/*.json",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
24
packages/sds/rollup.config.js
Normal file
24
packages/sds/rollup.config.js
Normal file
@ -0,0 +1,24 @@
|
||||
import commonjs from "@rollup/plugin-commonjs";
|
||||
import json from "@rollup/plugin-json";
|
||||
import { nodeResolve } from "@rollup/plugin-node-resolve";
|
||||
import { extractExports } from "@waku/build-utils";
|
||||
|
||||
import * as packageJson from "./package.json" assert { type: "json" };
|
||||
|
||||
const input = extractExports(packageJson);
|
||||
|
||||
export default {
|
||||
input,
|
||||
output: {
|
||||
dir: "bundle",
|
||||
format: "esm"
|
||||
},
|
||||
plugins: [
|
||||
commonjs(),
|
||||
json(),
|
||||
nodeResolve({
|
||||
browser: true,
|
||||
preferBuiltins: false
|
||||
})
|
||||
]
|
||||
};
|
||||
67
packages/sds/src/bloom.ts
Normal file
67
packages/sds/src/bloom.ts
Normal file
@ -0,0 +1,67 @@
|
||||
import { getMOverNBitsForK } from "./probabilities.js";
|
||||
|
||||
export interface BloomFilterOptions {
|
||||
// The expected maximum number of elements for which this BloomFilter is sized.
|
||||
capacity: number;
|
||||
|
||||
// The desired false-positive rate (between 0 and 1).
|
||||
errorRate: number;
|
||||
|
||||
// (Optional) The exact number of hash functions, if the user wants to override the automatic calculation.
|
||||
kHashes?: number;
|
||||
|
||||
// (Optional) Force a specific number of bits per element instead of using a table or optimal formula.
|
||||
forceNBitsPerElem?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A probabilistic data structure that tracks memberships in a set.
|
||||
* Supports time and space efficient lookups, but may return false-positives.
|
||||
* Can never return false-negatives.
|
||||
* A bloom filter can tell us if an element is:
|
||||
* - Definitely not in the set
|
||||
* - Potentially in the set (with a probability depending on the false-positive rate)
|
||||
*/
|
||||
export abstract class BloomFilter {
|
||||
public totalBits: number;
|
||||
public data: Uint8Array = new Uint8Array(0);
|
||||
|
||||
public constructor(options: BloomFilterOptions) {
|
||||
let nBitsPerElem: number;
|
||||
let k = options.kHashes ?? 0;
|
||||
const forceNBitsPerElem = options.forceNBitsPerElem ?? 0;
|
||||
|
||||
if (k < 1) {
|
||||
// Calculate optimal k based on target error rate
|
||||
const bitsPerElem = Math.ceil(
|
||||
-1.0 * (Math.log(options.errorRate) / Math.pow(Math.log(2), 2))
|
||||
);
|
||||
k = Math.round(Math.log(2) * bitsPerElem);
|
||||
nBitsPerElem = Math.round(bitsPerElem);
|
||||
} else {
|
||||
// Use specified k if possible
|
||||
if (forceNBitsPerElem < 1) {
|
||||
// Use lookup table
|
||||
nBitsPerElem = getMOverNBitsForK(k, options.errorRate);
|
||||
} else {
|
||||
nBitsPerElem = forceNBitsPerElem;
|
||||
}
|
||||
}
|
||||
|
||||
const mBits = options.capacity * nBitsPerElem;
|
||||
const mInts = 1 + mBits / (this.data.BYTES_PER_ELEMENT * 8);
|
||||
|
||||
this.totalBits = mBits;
|
||||
this.data = new Uint8Array(mInts);
|
||||
}
|
||||
|
||||
// Adds an item to the bloom filter by computing its hash values
|
||||
// and setting corresponding bits in "data".
|
||||
public abstract insert(item: string | Uint8Array): void;
|
||||
|
||||
// Checks if the item is potentially in the bloom filter.
|
||||
// The method is guaranteed to return "true" for items that were inserted,
|
||||
// but might also return "true" for items that were never inserted
|
||||
// (purpose of false-positive probability).
|
||||
public abstract lookup(item: string | Uint8Array): boolean;
|
||||
}
|
||||
9
packages/sds/src/index.spec.ts
Normal file
9
packages/sds/src/index.spec.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { expect } from "chai";
|
||||
|
||||
import { BloomFilter } from "./bloom.js";
|
||||
|
||||
describe("BloomFilter", () => {
|
||||
it("should be defined", () => {
|
||||
expect(BloomFilter).to.be.ok;
|
||||
});
|
||||
});
|
||||
3
packages/sds/src/index.ts
Normal file
3
packages/sds/src/index.ts
Normal file
@ -0,0 +1,3 @@
|
||||
import { BloomFilter } from "./bloom.js";
|
||||
|
||||
export { BloomFilter };
|
||||
166
packages/sds/src/probabilities.ts
Normal file
166
packages/sds/src/probabilities.ts
Normal file
@ -0,0 +1,166 @@
|
||||
// This file contains the probability tables used to determine the optimal number of
|
||||
// hash functions (k) and bits per element (m/n) for a Bloom filter.
|
||||
//
|
||||
// These are used to determine how to construct a Bloom filter that can perform
|
||||
// lookups with false-positive rate low enough to be satisfactory.
|
||||
|
||||
/**
|
||||
* Represents the error rates for a given number of hash functions (k) across
|
||||
* different (m/n) ratios (i.e., bits per element).
|
||||
*/
|
||||
type TErrorForK = Float32Array;
|
||||
|
||||
/**
|
||||
* An array where each index corresponds to a value of k (the number of hash functions),
|
||||
* and each element is a vector of false-positive rates for varying bits-per-element ratios.
|
||||
* Example:
|
||||
* ```ts
|
||||
* // Probability of a false positive upon lookup when using 1 hash function (k=1)
|
||||
* // and 15 bits per element (mOverN=15):
|
||||
* const falsePositiveRate = kErrors[1][15];
|
||||
* ```
|
||||
*/
|
||||
type TAllErrorRates = Array<TErrorForK>;
|
||||
|
||||
/**
|
||||
* Table of false positive rates for values of k from 0 to 12, and bits-per-element
|
||||
* ratios ranging from 0 up to around 32. Each Float32Array is indexed by mOverN,
|
||||
* so kErrors[k][mOverN] gives the estimated false-positive probability.
|
||||
*
|
||||
* These values mirror commonly used reference data found in Bloom filter literature,
|
||||
* such as:
|
||||
* https://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
|
||||
* https://dl.acm.org/doi/pdf/10.1145/362686.362692
|
||||
*/
|
||||
// prettier-ignore
|
||||
export const kErrors: TAllErrorRates = [
|
||||
new Float32Array([1.0]),
|
||||
new Float32Array([1.0, 1.0, 0.3930000000, 0.2830000000, 0.2210000000, 0.1810000000,
|
||||
0.1540000000, 0.1330000000, 0.1180000000, 0.1050000000, 0.0952000000,
|
||||
0.0869000000, 0.0800000000, 0.0740000000, 0.0689000000, 0.0645000000,
|
||||
0.0606000000, 0.0571000000, 0.0540000000, 0.0513000000, 0.0488000000,
|
||||
0.0465000000, 0.0444000000, 0.0425000000, 0.0408000000, 0.0392000000,
|
||||
0.0377000000, 0.0364000000, 0.0351000000, 0.0339000000, 0.0328000000,
|
||||
0.0317000000, 0.0308000000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 0.4000000000, 0.2370000000, 0.1550000000, 0.1090000000,
|
||||
0.0804000000, 0.0618000000, 0.0489000000, 0.0397000000, 0.0329000000,
|
||||
0.0276000000, 0.0236000000, 0.0203000000, 0.0177000000, 0.0156000000,
|
||||
0.0138000000, 0.0123000000, 0.0111000000, 0.0099800000, 0.0090600000,
|
||||
0.0082500000, 0.0075500000, 0.0069400000, 0.0063900000, 0.0059100000,
|
||||
0.0054800000, 0.0051000000, 0.0047500000, 0.0044400000, 0.0041600000,
|
||||
0.0039000000, 0.0036700000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 0.2530000000, 0.1470000000, 0.0920000000,
|
||||
0.0609000000, 0.0423000000, 0.0306000000, 0.0228000000, 0.0174000000,
|
||||
0.0136000000, 0.0108000000, 0.0087500000, 0.0071800000, 0.0059600000,
|
||||
0.0108000000, 0.0087500000, 0.0071800000, 0.0059600000, 0.0050000000,
|
||||
0.0042300000, 0.0036200000, 0.0031200000, 0.0027000000, 0.0023600000,
|
||||
0.0020700000, 0.0018300000, 0.0016200000, 0.0014500000, 0.0012900000,
|
||||
0.0011600000, 0.0010500000, 0.0009490000, 0.0008620000, 0.0007850000,
|
||||
0.0007170000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 0.1600000000, 0.0920000000, 0.0561000000, 0.0359000000,
|
||||
0.0240000000, 0.0166000000, 0.0118000000, 0.0086400000, 0.0064600000,
|
||||
0.0049200000, 0.0038100000, 0.0030000000, 0.0023900000, 0.0019300000,
|
||||
0.0015800000, 0.0013000000, 0.0010800000, 0.0009050000, 0.0007640000,
|
||||
0.0006490000, 0.0005550000, 0.0004780000, 0.0004130000, 0.0003590000,
|
||||
0.0003140000, 0.0002760000, 0.0002430000, 0.0002150000, 0.0001910000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 0.1010000000, 0.0578000000, 0.0347000000,
|
||||
0.0217000000, 0.0141000000, 0.0094300000, 0.0065000000, 0.0045900000,
|
||||
0.0033200000, 0.0024400000, 0.0018300000, 0.0013900000, 0.0010700000,
|
||||
0.0008390000, 0.0006630000, 0.0005300000, 0.0004270000, 0.0003470000,
|
||||
0.0002850000, 0.0002350000, 0.0001960000, 0.0001640000, 0.0001380000,
|
||||
0.0001170000, 0.0000996000, 0.0000853000, 0.0000733000, 0.0000633000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0638000000, 0.0364000000, 0.0216000000,
|
||||
0.0133000000, 0.0084400000, 0.0055200000, 0.0037100000, 0.0025500000,
|
||||
0.0017900000, 0.0012800000, 0.0009350000, 0.0006920000, 0.0005190000,
|
||||
0.0003940000, 0.0003030000, 0.0002360000, 0.0001850000, 0.0001470000,
|
||||
0.0001170000, 0.0000944000, 0.0000766000, 0.0000626000, 0.0000515000,
|
||||
0.0000426000, 0.0000355000, 0.0000297000, 0.0000250000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0229000000, 0.0135000000, 0.0081900000,
|
||||
0.0051300000, 0.0032900000, 0.0021700000, 0.0014600000, 0.0010000000,
|
||||
0.0007020000, 0.0004990000, 0.0003600000, 0.0002640000, 0.0001960000,
|
||||
0.0001470000, 0.0001120000, 0.0000856000, 0.0000663000, 0.0000518000,
|
||||
0.0000408000, 0.0000324000, 0.0000259000, 0.0000209000, 0.0000169000,
|
||||
0.0000138000, 0.0000113000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
|
||||
1.0, 0.0145000000, 0.0084600000, 0.0050900000, 0.0031400000, 0.0019900000,
|
||||
0.0012900000, 0.0008520000, 0.0005740000, 0.0003940000, 0.0002750000,
|
||||
0.0001940000, 0.0001400000, 0.0001010000, 0.0000746000, 0.0000555000,
|
||||
0.0000417000, 0.0000316000, 0.0000242000, 0.0000187000, 0.0000146000,
|
||||
0.0000114000, 0.0000090100, 0.0000071600, 0.0000057300]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0053100000, 0.0031700000,
|
||||
0.0019400000, 0.0012100000, 0.0007750000, 0.0005050000, 0.0003350000,
|
||||
0.0002260000, 0.0001550000, 0.0001080000, 0.0000759000, 0.0000542000,
|
||||
0.0000392000, 0.0000286000, 0.0000211000, 0.0000157000, 0.0000118000,
|
||||
0.0000089600, 0.0000068500, 0.0000052800, 0.0000041000, 0.0000032000]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0033400000,
|
||||
0.0019800000, 0.0012000000, 0.0007440000, 0.0004700000, 0.0003020000,
|
||||
0.0001980000, 0.0001320000, 0.0000889000, 0.0000609000, 0.0000423000,
|
||||
0.0000297000, 0.0000211000, 0.0000152000, 0.0000110000, 0.0000080700,
|
||||
0.0000059700, 0.0000044500, 0.0000033500, 0.0000025400, 0.0000019400]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
|
||||
0.0021000000, 0.0012400000, 0.0007470000, 0.0004590000, 0.0002870000,
|
||||
0.0001830000, 0.0001180000, 0.0000777000, 0.0000518000, 0.0000350000,
|
||||
0.0000240000, 0.0000166000, 0.0000116000, 0.0000082300, 0.0000058900,
|
||||
0.0000042500, 0.0000031000, 0.0000022800, 0.0000016900, 0.0000012600]),
|
||||
|
||||
new Float32Array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
|
||||
0.0007780000, 0.0004660000, 0.0002840000, 0.0001760000, 0.0001110000,
|
||||
0.0000712000, 0.0000463000, 0.0000305000, 0.0000204000, 0.0000138000,
|
||||
0.0000094200, 0.0000065200, 0.0000045600, 0.0000032200, 0.0000022900,
|
||||
0.0000016500, 0.0000012000, 0.0000008740]),
|
||||
]
|
||||
|
||||
/**
|
||||
* Given a number of hash functions (k) and a target false-positive rate (targetError),
|
||||
* determines the minimum (m/n) bits-per-element that satisfies the error threshold.
|
||||
*
|
||||
* In the context of a Bloom filter:
|
||||
* - m is the total number of bits in the filter.
|
||||
* - n is the number of elements you expect to insert.
|
||||
* Thus, (m/n) describes how many bits are assigned per inserted element.
|
||||
*
|
||||
* Example:
|
||||
* ```ts
|
||||
* // We want to use 3 hash functions (k=3) and a false-positive rate of 1% (targetError=0.01).
|
||||
* const mOverN = getMOverNBitsForK(3, 0.01);
|
||||
* // The function will iterate through the error tables and find the smallest m/n that satisfies the error threshold.
|
||||
* // In this case, kErrors[3][5] is the first value in the vector kErrors[3] that is less than 0.01 (0.0920000000).
|
||||
* console.log(mOverN); // 5
|
||||
* ```
|
||||
*
|
||||
* @param k - The number of hash functions.
|
||||
* @param targetError - The desired maximum false-positive rate.
|
||||
* @param probabilityTable - An optional table of false-positive probabilities indexed by k.
|
||||
* @returns The smallest (m/n) bit ratio for which the false-positive rate is below targetError.
|
||||
* @throws If k is out of range or if no suitable ratio can be found.
|
||||
*/
|
||||
export function getMOverNBitsForK(
|
||||
k: number,
|
||||
targetError: number,
|
||||
probabilityTable = kErrors
|
||||
): number {
|
||||
// Returns the optimal number of m/n bits for a given k.
|
||||
if (k < 0 || k > 12) {
|
||||
throw new Error("k must be <= 12.");
|
||||
}
|
||||
|
||||
for (let mOverN = 2; mOverN < probabilityTable[k].length; mOverN++) {
|
||||
if (probabilityTable[k][mOverN] < targetError) {
|
||||
return mOverN;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Specified value of k and error rate not achievable using less than 4 bytes / element."
|
||||
);
|
||||
}
|
||||
3
packages/sds/tsconfig.dev.json
Normal file
3
packages/sds/tsconfig.dev.json
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"extends": "../../tsconfig.dev"
|
||||
}
|
||||
10
packages/sds/tsconfig.json
Normal file
10
packages/sds/tsconfig.json
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "../../tsconfig",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist/",
|
||||
"rootDir": "src",
|
||||
"tsBuildInfoFile": "dist/.tsbuildinfo"
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["src/**/*.spec.ts", "src/test_utils"]
|
||||
}
|
||||
4
packages/sds/typedoc.json
Normal file
4
packages/sds/typedoc.json
Normal file
@ -0,0 +1,4 @@
|
||||
{
|
||||
"extends": ["../../typedoc.base.json"],
|
||||
"entryPoints": ["src/index.ts"]
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user