waku-org
/
js-waku
mirror of https://github.com/waku-org/js-waku.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge conflicts

This commit is contained in:
danisharora099 2023-09-02 16:28:42 +05:30
parent 0674a15c87 1c090924d0
commit c75bbc8353
No known key found for this signature in database
GPG Key ID: FBD2BF500037F135
19 changed files with 1098 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.github/workflows/ci.yml vendored
View File

@ -7,9 +7,16 @@ on:
- "staging"
- "trying"
pull_request:
workflow_dispatch:
inputs:
nim_wakunode_image:
description: "Docker hub image name taken from https://hub.docker.com/r/statusteam/nim-waku/tags. Format: statusteam/nim-waku:v0.19.0"
required: false
type: string
env:
NODE_JS: "18"
RETRY_ATTEMPTS: 3
jobs:
check:
@ -57,7 +64,7 @@ jobs:
node:
runs-on: ubuntu-latest
env:
WAKUNODE_IMAGE: "statusteam/nim-waku:v0.19.0"
WAKUNODE_IMAGE: ${{ github.event.inputs.nim_wakunode_image || 'statusteam/nim-waku:v0.19.0' }}
steps:
- uses: actions/checkout@v3
@ -66,9 +73,14 @@ jobs:
node-version: ${{ env.NODE_JS }}
- uses: ./.github/actions/npm
- run: npm run build:esm
- run: npm run test:node
env:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: ${{ env.RETRY_ATTEMPTS }}
command: npm run test:node
with: |
DEBUG: ""
- name: Upload debug logs on failure
@ -88,7 +100,8 @@ jobs:
node_optional:
runs-on: ubuntu-latest
env:
WAKUNODE_IMAGE: "statusteam/nim-waku:v0.19.0"
WAKUNODE_IMAGE: ${{ github.event.inputs.nim_wakunode_image || 'statusteam/nim-waku:v0.19.0' }}
steps:
- uses: actions/checkout@v3
@ -97,9 +110,14 @@ jobs:
node-version: ${{ env.NODE_JS }}
- uses: ./.github/actions/npm
- run: npm run build:esm
- run: npm run test:optional --workspace=@waku/tests
env:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: ${{ env.RETRY_ATTEMPTS }}
command: npm run test:optional --workspace=@waku/tests
with: |
DEBUG: ""
node_with_go_waku_master:
@ -117,9 +135,14 @@ jobs:
node-version: ${{ env.NODE_JS }}
- uses: ./.github/actions/npm
- run: npm run build:esm
- run: npm run test:node
env:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: ${{ env.RETRY_ATTEMPTS }}
command: npm run test:node
with: |
DEBUG: "waku:nwaku*,waku:test*"
- name: Upload debug logs on failure
@ -149,9 +172,14 @@ jobs:
node-version: ${{ env.NODE_JS }}
- uses: ./.github/actions/npm
- run: npm run build:esm
- run: npm run test:node
env:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: ${{ env.RETRY_ATTEMPTS }}
command: npm run test:node
with: |
DEBUG: "waku:nwaku*,waku:test*"
- name: Upload debug logs on failure

1392
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

12
packages/core/package.json
View File

@ -72,12 +72,12 @@
"node": ">=16"
},
"dependencies": {
"@noble/hashes": "^1.3.0",
"@noble/hashes": "^1.3.2",
"@waku/interfaces": "0.0.17",
"@waku/proto": "0.0.5",
"@waku/utils": "0.0.10",
"debug": "^4.3.4",
"it-all": "^3.0.2",
"it-all": "^3.0.3",
"it-length-prefixed": "^9.0.1",
"it-pipe": "^3.0.1",
"p-event": "^5.0.1",
@ -95,8 +95,8 @@
"@types/uuid": "^9.0.1",
"@waku/build-utils": "*",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"fast-check": "^3.8.1",
"cspell": "^7.0.1",
"fast-check": "^3.12.0",
"ignore-loader": "^0.1.2",
"isomorphic-fetch": "^3.0.0",
"karma": "^6.4.1",
@ -106,8 +106,8 @@
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"process": "^0.11.10",
"puppeteer": "^20.4.0",
"rollup": "^3.21.3",
"puppeteer": "^21.1.1",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"

77
packages/core/src/lib/light_push/index.ts
View File

@ -27,6 +27,16 @@ const log = debug("waku:light-push");
export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1";
export { PushResponse };
type PreparePushMessageResult =
| {
query: PushRpc;
error: null;
}
| {
query: null;
error: SendError;
};
/**
* Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/).
*/
@ -38,37 +48,64 @@ class LightPush extends BaseProtocol implements ILightPush {
this.options = options || {};
}
async send(
private async preparePushMessage(
encoder: IEncoder,
message: IMessage,
opts?: ProtocolOptions
): Promise<SendResult> {
const { pubSubTopic = DefaultPubSubTopic } = this.options;
const peer = await this.getPeer(opts?.peerId);
const stream = await this.newStream(peer);
const recipients: PeerId[] = [];
let error: undefined | SendError = undefined;
pubSubTopic: string
): Promise<PreparePushMessageResult> {
try {
if (!isSizeValid(message.payload)) {
log("Failed to send waku light push: message is bigger that 1MB");
return {
recipients,
error: SendError.SIZE_TOO_BIG
};
log("Failed to send waku light push: message is bigger than 1MB");
return { query: null, error: SendError.SIZE_TOO_BIG };
}
const protoMessage = await encoder.toProtoObj(message);
if (!protoMessage) {
log("Failed to encode to protoMessage, aborting push");
return {
recipients,
query: null,
error: SendError.ENCODE_FAILED
};
}
const query = PushRpc.createRequest(protoMessage, pubSubTopic);
return { query, error: null };
} catch (error) {
log("Failed to prepare push message", error);
return {
query: null,
error: SendError.GENERIC_FAIL
};
}
}
async send(
encoder: IEncoder,
message: IMessage,
opts?: ProtocolOptions
): Promise<SendResult> {
const { pubSubTopic = DefaultPubSubTopic } = this.options;
const recipients: PeerId[] = [];
const { query, error: preparationError } = await this.preparePushMessage(
encoder,
message,
pubSubTopic
);
if (preparationError || !query) {
return {
recipients,
error: preparationError
};
}
let error: undefined | SendError = undefined;
const peer = await this.getPeer(opts?.peerId);
const stream = await this.newStream(peer);
try {
const res = await pipe(
[query.encode()],
lp.encode,
@ -76,6 +113,7 @@ class LightPush extends BaseProtocol implements ILightPush {
lp.decode,
async (source) => await all(source)
);
try {
const bytes = new Uint8ArrayList();
res.forEach((chunk) => {
@ -98,9 +136,10 @@ class LightPush extends BaseProtocol implements ILightPush {
log("Failed to send waku light push request", err);
error = SendError.GENERIC_FAIL;
}
return {
error,
recipients
recipients,
error
};
}
}

93
packages/core/src/lib/store/index.ts
View File

@ -86,6 +86,54 @@ class Store extends BaseProtocol implements IStore {
this.options = options ?? {};
}
/**
* Processes messages based on the provided callback and options.
* @private
*/
private async processMessages<T extends IDecodedMessage>(
messages: Promise<T | undefined>[],
callback: (message: T) => Promise<void | boolean> | boolean | void,
options?: QueryOptions
): Promise<boolean> {
let abort = false;
const messagesOrUndef: Array<T | undefined> = await Promise.all(messages);
let processedMessages: Array<T> = messagesOrUndef.filter(isDefined);
if (this.shouldReverseOrder(options)) {
processedMessages = processedMessages.reverse();
}
await Promise.all(
processedMessages.map(async (msg) => {
if (msg && !abort) {
abort = Boolean(await callback(msg));
}
})
);
return abort;
}
/**
* Determines whether to reverse the order of messages based on the provided options.
*
* Messages in pages are ordered from oldest (first) to most recent (last).
* https://github.com/vacp2p/rfc/issues/533
*
* @private
*/
private shouldReverseOrder(options?: QueryOptions): boolean {
return (
typeof options?.pageDirection === "undefined" ||
options?.pageDirection === PageDirection.BACKWARD
);
}
/**
* @deprecated Use `queryWithOrderedCallback` instead
**/
queryOrderedCallback = this.queryWithOrderedCallback;
/**
* Do a query to a Waku Store to retrieve historical/missed messages.
*
@ -103,42 +151,20 @@ class Store extends BaseProtocol implements IStore {
* or if an error is encountered when processing the reply,
* or if two decoders with the same content topic are passed.
*/
async queryOrderedCallback<T extends IDecodedMessage>(
async queryWithOrderedCallback<T extends IDecodedMessage>(
decoders: IDecoder<T>[],
callback: (message: T) => Promise<void | boolean> | boolean | void,
options?: QueryOptions
): Promise<void> {
let abort = false;
for await (const promises of this.queryGenerator(decoders, options)) {
if (abort) break;
const messagesOrUndef: Array<T | undefined> = await Promise.all(promises);
let messages: Array<T> = messagesOrUndef.filter(isDefined);
// Messages in pages are ordered from oldest (first) to most recent (last).
// https://github.com/vacp2p/rfc/issues/533
if (
typeof options?.pageDirection === "undefined" ||
options?.pageDirection === PageDirection.BACKWARD
) {
messages = messages.reverse();
}
await Promise.all(
messages.map(async (msg) => {
if (msg && !abort) {
abort = Boolean(await callback(msg));
}
})
);
if (await this.processMessages(promises, callback, options)) break;
}
}
/**
* Do a query to a Waku Store to retrieve historical/missed messages.
*
* The callback function takes a `Promise<WakuMessage>` in input,
* useful if messages needs to be decrypted and performance matters.
* useful if messages need to be decrypted and performance matters.
*
* The order of the messages passed to the callback is as follows:
* - within a page, messages are expected to be ordered from oldest to most recent
@ -152,7 +178,7 @@ class Store extends BaseProtocol implements IStore {
* or if an error is encountered when processing the reply,
* or if two decoders with the same content topic are passed.
*/
async queryCallbackOnPromise<T extends IDecodedMessage>(
async queryWithPromiseCallback<T extends IDecodedMessage>(
decoders: IDecoder<T>[],
callback: (
message: Promise<T | undefined>
@ -160,17 +186,15 @@ class Store extends BaseProtocol implements IStore {
options?: QueryOptions
): Promise<void> {
let abort = false;
let promises: Promise<void>[] = [];
for await (const page of this.queryGenerator(decoders, options)) {
const _promises = page.map(async (msg) => {
if (!abort) {
abort = Boolean(await callback(msg));
}
const _promises = page.map(async (msgPromise) => {
if (abort) return;
abort = Boolean(await callback(msgPromise));
});
promises = promises.concat(_promises);
await Promise.all(_promises);
if (abort) break;
}
await Promise.all(promises);
}
/**
@ -183,9 +207,6 @@ class Store extends BaseProtocol implements IStore {
* as follows:
* - within a page, messages SHOULD be ordered from oldest to most recent
* - pages direction depends on { @link QueryOptions.pageDirection }
*
* However, there is no way to guarantee the behavior of the remote node.
*
* @throws If not able to reach a Waku Store peer to query,
* or if an error is encountered when processing the reply,
* or if two decoders with the same content topic are passed.

6
packages/dns-discovery/package.json
View File

@ -59,7 +59,7 @@
"uint8arrays": "^4.0.4"
},
"devDependencies": {
"@libp2p/peer-id": "^2.0.4",
"@libp2p/peer-id": "^3.0.2",
"@libp2p/peer-id-factory": "^3.0.3",
"@multiformats/multiaddr": "^12.0.0",
"@rollup/plugin-commonjs": "^24.0.1",
@ -69,14 +69,14 @@
"@waku/build-utils": "*",
"@waku/interfaces": "0.0.17",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"karma": "^6.4.1",
"karma-chrome-launcher": "^3.2.0",
"karma-mocha": "^2.0.1",
"karma-webpack": "^5.0.0",
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"rollup": "^3.21.3",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"typescript": "^5.0.4"
},

8
packages/enr/package.json
View File

@ -53,7 +53,7 @@
"dependencies": {
"@ethersproject/rlp": "^5.7.0",
"@libp2p/crypto": "^1.0.17",
"@libp2p/peer-id": "^2.0.4",
"@libp2p/peer-id": "^3.0.2",
"@multiformats/multiaddr": "^12.0.0",
"@noble/secp256k1": "^2.0.0",
"@waku/utils": "0.0.10",
@ -70,7 +70,7 @@
"@waku/build-utils": "*",
"@waku/interfaces": "0.0.17",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"karma": "^6.4.1",
"karma-chrome-launcher": "^3.2.0",
"karma-mocha": "^2.0.1",
@ -78,8 +78,8 @@
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"process": "^0.11.10",
"puppeteer": "^20.4.0",
"rollup": "^3.21.3",
"puppeteer": "^21.1.1",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"typescript": "^5.0.4",
"uint8arrays": "^4.0.4"

4
packages/interfaces/package.json
View File

@ -47,9 +47,9 @@
"node": ">=16"
},
"devDependencies": {
"@chainsafe/libp2p-gossipsub": "^10.0.0",
"@chainsafe/libp2p-gossipsub": "^10.1.0",
"@multiformats/multiaddr": "^12.0.0",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"npm-run-all": "^4.1.5",
"typescript": "^5.0.4",
"libp2p": "^0.46.3"

4
packages/interfaces/src/store.ts
View File

@ -46,12 +46,12 @@ export type StoreQueryOptions = {
} & ProtocolOptions;
export interface IStore extends IBaseProtocol {
queryOrderedCallback: <T extends IDecodedMessage>(
queryWithOrderedCallback: <T extends IDecodedMessage>(
decoders: IDecoder<T>[],
callback: (message: T) => Promise<void | boolean> | boolean | void,
options?: StoreQueryOptions
) => Promise<void>;
queryCallbackOnPromise: <T extends IDecodedMessage>(
queryWithPromiseCallback: <T extends IDecodedMessage>(
decoders: IDecoder<T>[],
callback: (
message: Promise<T | undefined>

8
packages/message-encryption/package.json
View File

@ -87,8 +87,8 @@
"@types/mocha": "^10.0.1",
"@waku/build-utils": "*",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"fast-check": "^3.8.1",
"cspell": "^7.0.1",
"fast-check": "^3.12.0",
"karma": "^6.4.1",
"karma-chrome-launcher": "^3.2.0",
"karma-mocha": "^2.0.1",
@ -96,8 +96,8 @@
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"process": "^0.11.10",
"puppeteer": "^20.4.0",
"rollup": "^3.21.3",
"puppeteer": "^21.1.1",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"typescript": "^5.0.4"
},

10
packages/message-hash/package.json
View File

@ -50,7 +50,7 @@
"node": ">=16"
},
"dependencies": {
"@noble/hashes": "^1.2.0",
"@noble/hashes": "^1.3.2",
"@waku/utils": "0.0.10"
},
"devDependencies": {
@ -63,8 +63,8 @@
"@waku/build-utils": "*",
"@waku/interfaces": "0.0.17",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"fast-check": "^3.7.0",
"cspell": "^7.0.1",
"fast-check": "^3.12.0",
"ignore-loader": "^0.1.2",
"isomorphic-fetch": "^3.0.0",
"karma": "^6.4.1",
@ -74,8 +74,8 @@
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"process": "^0.11.10",
"puppeteer": "^20.4.0",
"rollup": "^3.15.0",
"puppeteer": "^21.1.1",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"

6
packages/peer-exchange/package.json
View File

@ -55,7 +55,7 @@
"@waku/proto": "0.0.5",
"@waku/utils": "0.0.10",
"debug": "^4.3.4",
"it-all": "^3.0.2",
"it-all": "^3.0.3",
"it-length-prefixed": "^9.0.1",
"it-pipe": "^3.0.1"
},
@ -65,9 +65,9 @@
"@rollup/plugin-node-resolve": "^15.1.0",
"@waku/build-utils": "*",
"chai": "^4.3.7",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"npm-run-all": "^4.1.5",
"rollup": "^3.21.3",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"typescript": "^5.0.4",
"uint8arraylist": "^2.4.3"

4
packages/proto/package.json
View File

@ -51,10 +51,10 @@
"@rollup/plugin-json": "^6.0.0",
"@rollup/plugin-node-resolve": "^15.1.0",
"@waku/build-utils": "*",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"npm-run-all": "^4.1.5",
"protons": "^7.0.2",
"rollup": "^3.21.3",
"rollup": "^3.28.0",
"typescript": "^5.0.4",
"uint8arraylist": "^2.4.3"
},

8
packages/relay/package.json
View File

@ -49,22 +49,22 @@
"node": ">=16"
},
"dependencies": {
"@chainsafe/libp2p-gossipsub": "^10.0.0",
"@noble/hashes": "^1.3.0",
"@chainsafe/libp2p-gossipsub": "^10.1.0",
"@noble/hashes": "^1.3.2",
"@waku/core": "0.0.22",
"@waku/interfaces": "0.0.17",
"@waku/proto": "0.0.5",
"@waku/utils": "0.0.10",
"chai": "^4.3.7",
"debug": "^4.3.4",
"fast-check": "^3.8.1"
"fast-check": "^3.12.0"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^24.1.0",
"@waku/build-utils": "*",
"@rollup/plugin-json": "^6.0.0",
"@rollup/plugin-node-resolve": "^15.1.0",
"rollup": "^3.15.0",
"rollup": "^3.28.0",
"ts-loader": "^9.4.2",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"

6
packages/sdk/package.json
View File

@ -59,15 +59,15 @@
"libp2p": "^0.46.3"
},
"devDependencies": {
"@chainsafe/libp2p-gossipsub": "^10.0.0",
"@chainsafe/libp2p-gossipsub": "^10.1.0",
"@rollup/plugin-commonjs": "^24.0.1",
"@rollup/plugin-json": "^6.0.0",
"@rollup/plugin-node-resolve": "^15.1.0",
"@waku/build-utils": "*",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"interface-datastore": "^7.0.4",
"npm-run-all": "^4.1.5",
"rollup": "^3.21.3",
"rollup": "^3.28.0",
"typescript": "^5.0.4"
},
"typedoc": {

4
packages/tests/package.json
View File

@ -51,7 +51,7 @@
},
"dependencies": {
"@libp2p/interface-compliance-tests": "^4.0.2",
"@libp2p/peer-id": "^2.0.4",
"@libp2p/peer-id": "^3.0.2",
"@waku/core": "*",
"@waku/enr": "*",
"@waku/interfaces": "*",
@ -80,7 +80,7 @@
"@waku/peer-exchange": "*",
"chai": "^4.3.7",
"datastore-core": "^9.2.2",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"debug": "^4.3.4",
"interface-datastore": "^8.2.3",
"libp2p": "^0.46.3",

4
packages/tests/src/run-tests.js
View File

@ -9,9 +9,9 @@ const WAKUNODE_IMAGE =
async function main() {
try {
await execAsync(`docker inspect ${WAKUNODE_IMAGE}`);
console.log("Using local image");
console.log(`Using local image ${WAKUNODE_IMAGE}`);
} catch (error) {
console.log("Pulling image...");
console.log(`Pulling image ${WAKUNODE_IMAGE}`);
await execAsync(`docker pull ${WAKUNODE_IMAGE}`);
console.log("Image pulled");
}

14
packages/tests/tests/store.node.spec.ts
View File

@ -204,7 +204,7 @@ describe("Waku Store", () => {
await waitForRemotePeer(waku, [Protocols.Store]);
const messages: IMessage[] = [];
await waku.store.queryCallbackOnPromise(
await waku.store.queryWithPromiseCallback(
[TestDecoder],
async (msgPromise) => {
const msg = await msgPromise;
@ -246,7 +246,7 @@ describe("Waku Store", () => {
const desiredMsgs = 14;
const messages: IMessage[] = [];
await waku.store.queryCallbackOnPromise(
await waku.store.queryWithPromiseCallback(
[TestDecoder],
async (msgPromise) => {
const msg = await msgPromise;
@ -285,7 +285,7 @@ describe("Waku Store", () => {
await waitForRemotePeer(waku, [Protocols.Store]);
const messages: IMessage[] = [];
await waku.store.queryOrderedCallback(
await waku.store.queryWithOrderedCallback(
[TestDecoder],
async (msg) => {
messages.push(msg);
@ -324,7 +324,7 @@ describe("Waku Store", () => {
await waitForRemotePeer(waku, [Protocols.Store]);
let messages: IMessage[] = [];
await waku.store.queryOrderedCallback(
await waku.store.queryWithOrderedCallback(
[TestDecoder],
async (msg) => {
messages.push(msg);
@ -491,7 +491,7 @@ describe("Waku Store", () => {
const nwakuPeerId = await nwaku.getPeerId();
const firstMessages: IMessage[] = [];
await waku.store.queryOrderedCallback(
await waku.store.queryWithOrderedCallback(
[TestDecoder],
(msg) => {
if (msg) {
@ -505,7 +505,7 @@ describe("Waku Store", () => {
);
const bothMessages: IMessage[] = [];
await waku.store.queryOrderedCallback(
await waku.store.queryWithOrderedCallback(
[TestDecoder],
async (msg) => {
bothMessages.push(msg);
@ -552,7 +552,7 @@ describe("Waku Store", () => {
const desiredMsgs = 14;
const messages: IMessage[] = [];
await waku.store.queryOrderedCallback(
await waku.store.queryWithOrderedCallback(
[TestDecoder],
async (msg) => {
messages.push(msg);

4
packages/utils/package.json
View File

@ -74,9 +74,9 @@
"@rollup/plugin-node-resolve": "^15.1.0",
"@waku/build-utils": "*",
"@waku/interfaces": "0.0.17",
"cspell": "^7.0.0",
"cspell": "^7.0.1",
"npm-run-all": "^4.1.5",
"rollup": "^3.21.3",
"rollup": "^3.28.0",
"typescript": "^5.0.4"
},
"typedoc": {