status-im
/
js-waku
mirror of https://github.com/status-im/js-waku.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: add bootstrapPeers option and refactor sdk (#1871) 

* move relay related code

* move libp2p to utils

* define CreateWakuNodeOptions

* improve options

* make libp2p use from create function

* add bootstrapPeers option

* fix lint

* fix types, imports

* fix exports

* use bootstrap along default bootstrap

* fix test as REST does not return peer though connection is made

* rollback condition change

* enable gossipSub back for every node
This commit is contained in:
Sasha 2024-03-04 10:56:20 +01:00 committed by GitHub
parent fcc3f10f7c
commit 9f198dd149
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 212 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
package-lock.json generated
View File

@ -2714,7 +2714,6 @@
"version": "10.0.11",
"resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz",
"integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==",
"dev": true,
"dependencies": {
"@libp2p/interface": "^1.1.1",
"@libp2p/peer-id": "^4.0.4",
@ -28811,6 +28810,7 @@
"license": "MIT OR Apache-2.0",
"dependencies": {
"@chainsafe/libp2p-noise": "^14.1.0",
"@libp2p/bootstrap": "^10.0.11",
"@libp2p/identify": "^1.0.11",
"@libp2p/mplex": "^10.0.12",
"@libp2p/ping": "^1.0.11",
@ -28837,6 +28837,9 @@
},
"engines": {
"node": ">=18"
},
"peerDependencies": {
"@libp2p/bootstrap": "^10"
}
},
"packages/tests": {
@ -30748,7 +30751,6 @@
"version": "10.0.11",
"resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz",
"integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==",
"dev": true,
"requires": {
"@libp2p/interface": "^1.1.1",
"@libp2p/peer-id": "^4.0.4",
@ -33278,6 +33280,7 @@
"requires": {
"@chainsafe/libp2p-gossipsub": "^12.0.0",
"@chainsafe/libp2p-noise": "^14.1.0",
"@libp2p/bootstrap": "^10.0.11",
"@libp2p/identify": "^1.0.11",
"@libp2p/mplex": "^10.0.12",
"@libp2p/ping": "^1.0.11",

4
packages/interfaces/src/protocols.ts
View File

@ -87,6 +87,10 @@ export type ProtocolCreateOptions = {
* Use recommended bootstrap method to discovery and connect to new nodes.
*/
defaultBootstrap?: boolean;
/**
* List of peers to use to bootstrap the node. Ignored if defaultBootstrap is set to true.
*/
bootstrapPeers?: string[];
};
export type Callback<T extends IDecodedMessage> = (

6
packages/sdk/package.json
View File

@ -73,7 +73,8 @@
"@waku/peer-exchange": "^0.0.19",
"@waku/relay": "0.0.9",
"@waku/utils": "0.0.14",
"libp2p": "^1.1.2"
"libp2p": "^1.1.2",
"@libp2p/bootstrap": "^10.0.11"
},
"devDependencies": {
"@chainsafe/libp2p-gossipsub": "^12.0.0",
@ -86,6 +87,9 @@
"npm-run-all": "^4.1.5",
"rollup": "^4.12.0"
},
"peerDependencies": {
"@libp2p/bootstrap": "^10"
},
"files": [
"dist",
"bundle",

211
packages/sdk/src/create.ts
View File

@ -1,38 +1,8 @@
import type { GossipSub } from "@chainsafe/libp2p-gossipsub";
import { noise } from "@chainsafe/libp2p-noise";
import { identify } from "@libp2p/identify";
import type { PeerDiscovery } from "@libp2p/interface";
import { mplex } from "@libp2p/mplex";
import { ping } from "@libp2p/ping";
import { webSockets } from "@libp2p/websockets";
import { all as filterAll } from "@libp2p/websockets/filters";
import { wakuFilter, wakuLightPush, wakuMetadata, wakuStore } from "@waku/core";
import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
import {
type CreateLibp2pOptions,
DefaultPubsubTopic,
type FullNode,
type IMetadata,
type Libp2p,
type Libp2pComponents,
type LightNode,
type ProtocolCreateOptions,
PubsubTopic,
type ShardInfo
} from "@waku/interfaces";
import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery";
import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange";
import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
import { ensureShardingConfigured } from "@waku/utils";
import { createLibp2p } from "libp2p";
import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core";
import { type Libp2pComponents, type LightNode } from "@waku/interfaces";
import { DefaultUserAgent, WakuNode, WakuOptions } from "./waku.js";
const DEFAULT_NODE_REQUIREMENTS = {
lightPush: 1,
filter: 1,
store: 1
};
import { createLibp2pAndUpdateOptions } from "./utils/libp2p.js";
import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "./waku.js";
export { Libp2pComponents };
@ -40,33 +10,13 @@ export { Libp2pComponents };
* Create a Waku node configured to use autosharding or static sharding.
*/
export async function createNode(
options?: ProtocolCreateOptions &
Partial<WakuOptions> &
Partial<RelayCreateOptions>
options: CreateWakuNodeOptions = { pubsubTopics: [] }
): Promise<LightNode> {
options = options ?? { pubsubTopics: [] };
if (!options.shardInfo) {
throw new Error("Shard info must be set");
}
const shardInfo = ensureShardingConfigured(options.shardInfo);
options.pubsubTopics = shardInfo.pubsubTopics;
options.shardInfo = shardInfo.shardInfo;
const libp2pOptions = options?.libp2p ?? {};
const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
if (options?.defaultBootstrap) {
peerDiscovery.push(...defaultPeerDiscoveries(shardInfo.pubsubTopics));
Object.assign(libp2pOptions, { peerDiscovery });
}
const libp2p = await defaultLibp2p(
shardInfo.shardInfo,
wakuGossipSub(options),
libp2pOptions,
options?.userAgent
);
const libp2p = await createLibp2pAndUpdateOptions(options);
const store = wakuStore(options);
const lightPush = wakuLightPush(options);
@ -87,30 +37,9 @@ export async function createNode(
* Uses Waku Filter V2 by default.
*/
export async function createLightNode(
options?: ProtocolCreateOptions & Partial<WakuOptions>
options: CreateWakuNodeOptions = {}
): Promise<LightNode> {
options = options ?? {};
const shardInfo = options.shardInfo
? ensureShardingConfigured(options.shardInfo)
: undefined;
options.pubsubTopics = shardInfo?.pubsubTopics ??
options.pubsubTopics ?? [DefaultPubsubTopic];
const libp2pOptions = options?.libp2p ?? {};
const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
if (options?.defaultBootstrap) {
peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
Object.assign(libp2pOptions, { peerDiscovery });
}
const libp2p = await defaultLibp2p(
shardInfo?.shardInfo,
wakuGossipSub(options),
libp2pOptions,
options?.userAgent
);
const libp2p = await createLibp2pAndUpdateOptions(options);
const store = wakuStore(options);
const lightPush = wakuLightPush(options);
@ -124,127 +53,3 @@ export async function createLightNode(
filter
) as LightNode;
}
/**
* Create a Waku node that uses all Waku protocols.
*
* This helper is not recommended except if:
* - you are interfacing with nwaku v0.11 or below
* - you are doing some form of testing
*
* If you are building a full node, it is recommended to use
* [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API.
*
* @see https://github.com/status-im/nwaku/issues/1085
* @internal
*/
export async function createFullNode(
options?: ProtocolCreateOptions &
Partial<WakuOptions> &
Partial<RelayCreateOptions>
): Promise<FullNode> {
options = options ?? { pubsubTopics: [] };
const shardInfo = options.shardInfo
? ensureShardingConfigured(options.shardInfo)
: undefined;
const pubsubTopics = shardInfo?.pubsubTopics ??
options.pubsubTopics ?? [DefaultPubsubTopic];
options.pubsubTopics = pubsubTopics;
options.shardInfo = shardInfo?.shardInfo;
const libp2pOptions = options?.libp2p ?? {};
const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
if (options?.defaultBootstrap) {
peerDiscovery.push(...defaultPeerDiscoveries(pubsubTopics));
Object.assign(libp2pOptions, { peerDiscovery });
}
const libp2p = await defaultLibp2p(
shardInfo?.shardInfo,
wakuGossipSub(options),
libp2pOptions,
options?.userAgent
);
const store = wakuStore(options);
const lightPush = wakuLightPush(options);
const filter = wakuFilter(options);
const relay = wakuRelay(pubsubTopics);
return new WakuNode(
options as WakuOptions,
libp2p,
store,
lightPush,
filter,
relay
) as FullNode;
}
export function defaultPeerDiscoveries(
pubsubTopics: PubsubTopic[]
): ((components: Libp2pComponents) => PeerDiscovery)[] {
const discoveries = [
wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS),
wakuLocalPeerCacheDiscovery(),
wakuPeerExchangeDiscovery(pubsubTopics)
];
return discoveries;
}
type PubsubService = {
pubsub?: (components: Libp2pComponents) => GossipSub;
};
type MetadataService = {
metadata?: (components: Libp2pComponents) => IMetadata;
};
export async function defaultLibp2p(
shardInfo?: ShardInfo,
wakuGossipSub?: PubsubService["pubsub"],
options?: Partial<CreateLibp2pOptions>,
userAgent?: string
): Promise<Libp2p> {
if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") {
/* eslint-disable no-console */
console.info(
"%cIgnore WebSocket connection failures",
"background: gray; color: white; font-size: x-large"
);
console.info(
"%cWaku tries to discover peers and some of them are expected to fail",
"background: gray; color: white; font-size: x-large"
);
/* eslint-enable no-console */
}
const pubsubService: PubsubService = wakuGossipSub
? { pubsub: wakuGossipSub }
: {};
const metadataService: MetadataService = shardInfo
? { metadata: wakuMetadata(shardInfo) }
: {};
return createLibp2p({
connectionManager: {
minConnections: 1
},
transports: [webSockets({ filter: filterAll })],
streamMuxers: [mplex()],
connectionEncryption: [noise()],
...options,
services: {
identify: identify({
agentVersion: userAgent ?? DefaultUserAgent
}),
ping: ping(),
...metadataService,
...pubsubService,
...options?.services
}
}) as any as Libp2p; // TODO: make libp2p include it;
}

3
packages/sdk/src/index.ts
View File

@ -7,7 +7,8 @@ export {
export { utf8ToBytes, bytesToUtf8 } from "@waku/utils/bytes";
export * from "./content_topic.js";
export { defaultLibp2p } from "./utils/libp2p.js";
export * from "./utils/content_topic.js";
export * from "./waku.js";
export * from "./create.js";
export * as waku from "@waku/core";

81
packages/sdk/src/relay/index.ts
View File

@ -1,13 +1,9 @@
import {
DefaultPubsubTopic,
type ProtocolCreateOptions,
type RelayNode
} from "@waku/interfaces";
import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
import { ensureShardingConfigured } from "@waku/utils";
import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core";
import { type FullNode, type RelayNode } from "@waku/interfaces";
import { RelayCreateOptions, wakuRelay } from "@waku/relay";
import { defaultLibp2p, defaultPeerDiscoveries } from "../create.js";
import { WakuNode, WakuOptions } from "../waku.js";
import { createLibp2pAndUpdateOptions } from "../utils/libp2p.js";
import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "../waku.js";
/**
* Create a Waku node that uses Waku Relay to send and receive messages,
@ -20,35 +16,13 @@ import { WakuNode, WakuOptions } from "../waku.js";
* or use this function with caution.
*/
export async function createRelayNode(
options?: ProtocolCreateOptions &
Partial<WakuOptions> &
Partial<RelayCreateOptions>
): Promise<RelayNode> {
options = options ?? { pubsubTopics: [] };
const libp2pOptions = options?.libp2p ?? {};
const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
const shardInfo = options.shardInfo
? ensureShardingConfigured(options.shardInfo)
: undefined;
options.pubsubTopics = shardInfo?.pubsubTopics ??
options.pubsubTopics ?? [DefaultPubsubTopic];
if (options?.defaultBootstrap) {
peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
Object.assign(libp2pOptions, { peerDiscovery });
options: CreateWakuNodeOptions & Partial<RelayCreateOptions> = {
pubsubTopics: []
}
): Promise<RelayNode> {
const libp2p = await createLibp2pAndUpdateOptions(options);
const libp2p = await defaultLibp2p(
shardInfo?.shardInfo,
wakuGossipSub(options),
libp2pOptions,
options?.userAgent
);
const relay = wakuRelay(options.pubsubTopics);
const relay = wakuRelay(options?.pubsubTopics || []);
return new WakuNode(
options as WakuOptions,
@ -59,3 +33,38 @@ export async function createRelayNode(
relay
) as RelayNode;
}
/**
* Create a Waku node that uses all Waku protocols.
*
* This helper is not recommended except if:
* - you are interfacing with nwaku v0.11 or below
* - you are doing some form of testing
*
* If you are building a full node, it is recommended to use
* [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API.
*
* @see https://github.com/status-im/nwaku/issues/1085
* @internal
*/
export async function createFullNode(
options: CreateWakuNodeOptions & Partial<RelayCreateOptions> = {
pubsubTopics: []
}
): Promise<FullNode> {
const libp2p = await createLibp2pAndUpdateOptions(options);
const store = wakuStore(options);
const lightPush = wakuLightPush(options);
const filter = wakuFilter(options);
const relay = wakuRelay(options?.pubsubTopics || []);
return new WakuNode(
options as WakuOptions,
libp2p,
store,
lightPush,
filter,
relay
) as FullNode;
}

2
packages/sdk/src/content_topic.ts → packages/sdk/src/utils/content_topic.ts
View File

@ -12,7 +12,7 @@ import {
shardInfoToPubsubTopics
} from "@waku/utils";
import { createLightNode } from "./create.js";
import { createLightNode } from "../create.js";
interface CreateTopicOptions {
waku?: LightNode;

22
packages/sdk/src/utils/discovery.ts Normal file
View File

@ -0,0 +1,22 @@
import type { PeerDiscovery } from "@libp2p/interface";
import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
import { type Libp2pComponents, PubsubTopic } from "@waku/interfaces";
import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery";
import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange";
const DEFAULT_NODE_REQUIREMENTS = {
lightPush: 1,
filter: 1,
store: 1
};
export function defaultPeerDiscoveries(
pubsubTopics: PubsubTopic[]
): ((components: Libp2pComponents) => PeerDiscovery)[] {
const discoveries = [
wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS),
wakuLocalPeerCacheDiscovery(),
wakuPeerExchangeDiscovery(pubsubTopics)
];
return discoveries;
}

112
packages/sdk/src/utils/libp2p.ts Normal file
View File

@ -0,0 +1,112 @@
import type { GossipSub } from "@chainsafe/libp2p-gossipsub";
import { noise } from "@chainsafe/libp2p-noise";
import { bootstrap } from "@libp2p/bootstrap";
import { identify } from "@libp2p/identify";
import { mplex } from "@libp2p/mplex";
import { ping } from "@libp2p/ping";
import { webSockets } from "@libp2p/websockets";
import { all as filterAll } from "@libp2p/websockets/filters";
import { wakuMetadata } from "@waku/core";
import {
type CreateLibp2pOptions,
DefaultPubsubTopic,
type IMetadata,
type Libp2p,
type Libp2pComponents,
type ShardInfo
} from "@waku/interfaces";
import { wakuGossipSub } from "@waku/relay";
import { ensureShardingConfigured } from "@waku/utils";
import { createLibp2p } from "libp2p";
import { CreateWakuNodeOptions, DefaultUserAgent } from "../waku.js";
import { defaultPeerDiscoveries } from "./discovery.js";
type PubsubService = {
pubsub?: (components: Libp2pComponents) => GossipSub;
};
type MetadataService = {
metadata?: (components: Libp2pComponents) => IMetadata;
};
export async function defaultLibp2p(
shardInfo?: ShardInfo,
wakuGossipSub?: PubsubService["pubsub"],
options?: Partial<CreateLibp2pOptions>,
userAgent?: string
): Promise<Libp2p> {
if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") {
/* eslint-disable no-console */
console.info(
"%cIgnore WebSocket connection failures",
"background: gray; color: white; font-size: x-large"
);
console.info(
"%cWaku tries to discover peers and some of them are expected to fail",
"background: gray; color: white; font-size: x-large"
);
/* eslint-enable no-console */
}
const pubsubService: PubsubService = wakuGossipSub
? { pubsub: wakuGossipSub }
: {};
const metadataService: MetadataService = shardInfo
? { metadata: wakuMetadata(shardInfo) }
: {};
return createLibp2p({
connectionManager: {
minConnections: 1
},
transports: [webSockets({ filter: filterAll })],
streamMuxers: [mplex()],
connectionEncryption: [noise()],
...options,
services: {
identify: identify({
agentVersion: userAgent ?? DefaultUserAgent
}),
ping: ping(),
...metadataService,
...pubsubService,
...options?.services
}
}) as any as Libp2p; // TODO: make libp2p include it;
}
export async function createLibp2pAndUpdateOptions(
options: CreateWakuNodeOptions
): Promise<Libp2p> {
const shardInfo = options.shardInfo
? ensureShardingConfigured(options.shardInfo)
: undefined;
options.pubsubTopics = shardInfo?.pubsubTopics ??
options.pubsubTopics ?? [DefaultPubsubTopic];
const libp2pOptions = options?.libp2p ?? {};
const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
if (options?.defaultBootstrap) {
peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
}
if (options?.bootstrapPeers) {
peerDiscovery.push(bootstrap({ list: options.bootstrapPeers }));
}
libp2pOptions.peerDiscovery = peerDiscovery;
const libp2p = await defaultLibp2p(
shardInfo?.shardInfo,
wakuGossipSub(options),
libp2pOptions,
options?.userAgent
);
return libp2p;
}

6
packages/sdk/src/waku.ts
View File

@ -11,13 +11,14 @@ import type {
IStore,
Libp2p,
LightNode,
ProtocolCreateOptions,
PubsubTopic,
Waku
} from "@waku/interfaces";
import { Protocols } from "@waku/interfaces";
import { Logger } from "@waku/utils";
import { subscribeToContentTopic } from "./content_topic.js";
import { subscribeToContentTopic } from "./utils/content_topic.js";
export const DefaultPingKeepAliveValueSecs = 5 * 60;
export const DefaultRelayKeepAliveValueSecs = 5 * 60;
@ -48,6 +49,9 @@ export interface WakuOptions {
pubsubTopics: PubsubTopic[];
}
export type CreateWakuNodeOptions = ProtocolCreateOptions &
Partial<WakuOptions>;
export class WakuNode implements Waku {
public libp2p: Libp2p;
public relay?: IRelay;

6
packages/tests/src/lib/service_node.ts
View File

@ -201,9 +201,9 @@ export class ServiceNode {
return waitForLine(this.logPath, msg, timeout);
}
/** Calls nwaku JSON-RPC API `get_waku_v2_admin_v1_peers` to check
* for known peers
* @throws if WakuNode isn't started.
/**
* Calls nwaku REST API "/admin/v1/peers" to check for known peers
* @throws
*/
async peers(): Promise<string[]> {
this.checkProcess();