From 9f198dd149ef299e3edce69b93cc2942c6f24846 Mon Sep 17 00:00:00 2001 From: Sasha <118575614+weboko@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:56:20 +0100 Subject: [PATCH] feat: add bootstrapPeers option and refactor sdk (#1871) * move relay related code * move libp2p to utils * define CreateWakuNodeOptions * improve options * make libp2p use from create function * add bootstrapPeers option * fix lint * fix types, imports * fix exports * use bootstrap along default bootstrap * fix test as REST does not return peer though connection is made * rollback condition change * enable gossipSub back for every node --- package-lock.json | 7 +- packages/interfaces/src/protocols.ts | 4 + packages/sdk/package.json | 6 +- packages/sdk/src/create.ts | 211 +----------------- packages/sdk/src/index.ts | 3 +- packages/sdk/src/relay/index.ts | 81 ++++--- packages/sdk/src/{ => utils}/content_topic.ts | 2 +- packages/sdk/src/utils/discovery.ts | 22 ++ packages/sdk/src/utils/libp2p.ts | 112 ++++++++++ packages/sdk/src/waku.ts | 6 +- packages/tests/src/lib/service_node.ts | 6 +- 11 files changed, 212 insertions(+), 248 deletions(-) rename packages/sdk/src/{ => utils}/content_topic.ts (98%) create mode 100644 packages/sdk/src/utils/discovery.ts create mode 100644 packages/sdk/src/utils/libp2p.ts diff --git a/package-lock.json b/package-lock.json index a4be813043..c7f105b228 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2714,7 +2714,6 @@ "version": "10.0.11", "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz", "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==", - "dev": true, "dependencies": { "@libp2p/interface": "^1.1.1", "@libp2p/peer-id": "^4.0.4", @@ -28811,6 +28810,7 @@ "license": "MIT OR Apache-2.0", "dependencies": { "@chainsafe/libp2p-noise": "^14.1.0", + "@libp2p/bootstrap": "^10.0.11", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.11", @@ -28837,6 +28837,9 @@ }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@libp2p/bootstrap": "^10" } }, "packages/tests": { @@ -30748,7 +30751,6 @@ "version": "10.0.11", "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz", "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==", - "dev": true, "requires": { "@libp2p/interface": "^1.1.1", "@libp2p/peer-id": "^4.0.4", @@ -33278,6 +33280,7 @@ "requires": { "@chainsafe/libp2p-gossipsub": "^12.0.0", "@chainsafe/libp2p-noise": "^14.1.0", + "@libp2p/bootstrap": "^10.0.11", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.11", diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index 9b3dedfcab..0d89834bf8 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -87,6 +87,10 @@ export type ProtocolCreateOptions = { * Use recommended bootstrap method to discovery and connect to new nodes. */ defaultBootstrap?: boolean; + /** + * List of peers to use to bootstrap the node. Ignored if defaultBootstrap is set to true. + */ + bootstrapPeers?: string[]; }; export type Callback = ( diff --git a/packages/sdk/package.json b/packages/sdk/package.json index c6f6bf44a5..e6b7cd34dd 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -73,7 +73,8 @@ "@waku/peer-exchange": "^0.0.19", "@waku/relay": "0.0.9", "@waku/utils": "0.0.14", - "libp2p": "^1.1.2" + "libp2p": "^1.1.2", + "@libp2p/bootstrap": "^10.0.11" }, "devDependencies": { "@chainsafe/libp2p-gossipsub": "^12.0.0", @@ -86,6 +87,9 @@ "npm-run-all": "^4.1.5", "rollup": "^4.12.0" }, + "peerDependencies": { + "@libp2p/bootstrap": "^10" + }, "files": [ "dist", "bundle", diff --git a/packages/sdk/src/create.ts b/packages/sdk/src/create.ts index 25d5ef6826..ba3e6d9ab9 100644 --- a/packages/sdk/src/create.ts +++ b/packages/sdk/src/create.ts @@ -1,38 +1,8 @@ -import type { GossipSub } from "@chainsafe/libp2p-gossipsub"; -import { noise } from "@chainsafe/libp2p-noise"; -import { identify } from "@libp2p/identify"; -import type { PeerDiscovery } from "@libp2p/interface"; -import { mplex } from "@libp2p/mplex"; -import { ping } from "@libp2p/ping"; -import { webSockets } from "@libp2p/websockets"; -import { all as filterAll } from "@libp2p/websockets/filters"; -import { wakuFilter, wakuLightPush, wakuMetadata, wakuStore } from "@waku/core"; -import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery"; -import { - type CreateLibp2pOptions, - DefaultPubsubTopic, - type FullNode, - type IMetadata, - type Libp2p, - type Libp2pComponents, - type LightNode, - type ProtocolCreateOptions, - PubsubTopic, - type ShardInfo -} from "@waku/interfaces"; -import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery"; -import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange"; -import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay"; -import { ensureShardingConfigured } from "@waku/utils"; -import { createLibp2p } from "libp2p"; +import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core"; +import { type Libp2pComponents, type LightNode } from "@waku/interfaces"; -import { DefaultUserAgent, WakuNode, WakuOptions } from "./waku.js"; - -const DEFAULT_NODE_REQUIREMENTS = { - lightPush: 1, - filter: 1, - store: 1 -}; +import { createLibp2pAndUpdateOptions } from "./utils/libp2p.js"; +import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "./waku.js"; export { Libp2pComponents }; @@ -40,33 +10,13 @@ export { Libp2pComponents }; * Create a Waku node configured to use autosharding or static sharding. */ export async function createNode( - options?: ProtocolCreateOptions & - Partial & - Partial + options: CreateWakuNodeOptions = { pubsubTopics: [] } ): Promise { - options = options ?? { pubsubTopics: [] }; - if (!options.shardInfo) { throw new Error("Shard info must be set"); } - const shardInfo = ensureShardingConfigured(options.shardInfo); - options.pubsubTopics = shardInfo.pubsubTopics; - options.shardInfo = shardInfo.shardInfo; - - const libp2pOptions = options?.libp2p ?? {}; - const peerDiscovery = libp2pOptions.peerDiscovery ?? []; - if (options?.defaultBootstrap) { - peerDiscovery.push(...defaultPeerDiscoveries(shardInfo.pubsubTopics)); - Object.assign(libp2pOptions, { peerDiscovery }); - } - - const libp2p = await defaultLibp2p( - shardInfo.shardInfo, - wakuGossipSub(options), - libp2pOptions, - options?.userAgent - ); + const libp2p = await createLibp2pAndUpdateOptions(options); const store = wakuStore(options); const lightPush = wakuLightPush(options); @@ -87,30 +37,9 @@ export async function createNode( * Uses Waku Filter V2 by default. */ export async function createLightNode( - options?: ProtocolCreateOptions & Partial + options: CreateWakuNodeOptions = {} ): Promise { - options = options ?? {}; - - const shardInfo = options.shardInfo - ? ensureShardingConfigured(options.shardInfo) - : undefined; - - options.pubsubTopics = shardInfo?.pubsubTopics ?? - options.pubsubTopics ?? [DefaultPubsubTopic]; - - const libp2pOptions = options?.libp2p ?? {}; - const peerDiscovery = libp2pOptions.peerDiscovery ?? []; - if (options?.defaultBootstrap) { - peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics)); - Object.assign(libp2pOptions, { peerDiscovery }); - } - - const libp2p = await defaultLibp2p( - shardInfo?.shardInfo, - wakuGossipSub(options), - libp2pOptions, - options?.userAgent - ); + const libp2p = await createLibp2pAndUpdateOptions(options); const store = wakuStore(options); const lightPush = wakuLightPush(options); @@ -124,127 +53,3 @@ export async function createLightNode( filter ) as LightNode; } - -/** - * Create a Waku node that uses all Waku protocols. - * - * This helper is not recommended except if: - * - you are interfacing with nwaku v0.11 or below - * - you are doing some form of testing - * - * If you are building a full node, it is recommended to use - * [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API. - * - * @see https://github.com/status-im/nwaku/issues/1085 - * @internal - */ -export async function createFullNode( - options?: ProtocolCreateOptions & - Partial & - Partial -): Promise { - options = options ?? { pubsubTopics: [] }; - - const shardInfo = options.shardInfo - ? ensureShardingConfigured(options.shardInfo) - : undefined; - - const pubsubTopics = shardInfo?.pubsubTopics ?? - options.pubsubTopics ?? [DefaultPubsubTopic]; - options.pubsubTopics = pubsubTopics; - options.shardInfo = shardInfo?.shardInfo; - - const libp2pOptions = options?.libp2p ?? {}; - const peerDiscovery = libp2pOptions.peerDiscovery ?? []; - if (options?.defaultBootstrap) { - peerDiscovery.push(...defaultPeerDiscoveries(pubsubTopics)); - Object.assign(libp2pOptions, { peerDiscovery }); - } - - const libp2p = await defaultLibp2p( - shardInfo?.shardInfo, - wakuGossipSub(options), - libp2pOptions, - options?.userAgent - ); - - const store = wakuStore(options); - const lightPush = wakuLightPush(options); - const filter = wakuFilter(options); - const relay = wakuRelay(pubsubTopics); - - return new WakuNode( - options as WakuOptions, - libp2p, - store, - lightPush, - filter, - relay - ) as FullNode; -} - -export function defaultPeerDiscoveries( - pubsubTopics: PubsubTopic[] -): ((components: Libp2pComponents) => PeerDiscovery)[] { - const discoveries = [ - wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS), - wakuLocalPeerCacheDiscovery(), - wakuPeerExchangeDiscovery(pubsubTopics) - ]; - return discoveries; -} - -type PubsubService = { - pubsub?: (components: Libp2pComponents) => GossipSub; -}; - -type MetadataService = { - metadata?: (components: Libp2pComponents) => IMetadata; -}; - -export async function defaultLibp2p( - shardInfo?: ShardInfo, - wakuGossipSub?: PubsubService["pubsub"], - options?: Partial, - userAgent?: string -): Promise { - if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") { - /* eslint-disable no-console */ - console.info( - "%cIgnore WebSocket connection failures", - "background: gray; color: white; font-size: x-large" - ); - console.info( - "%cWaku tries to discover peers and some of them are expected to fail", - "background: gray; color: white; font-size: x-large" - ); - /* eslint-enable no-console */ - } - - const pubsubService: PubsubService = wakuGossipSub - ? { pubsub: wakuGossipSub } - : {}; - - const metadataService: MetadataService = shardInfo - ? { metadata: wakuMetadata(shardInfo) } - : {}; - - return createLibp2p({ - connectionManager: { - minConnections: 1 - }, - transports: [webSockets({ filter: filterAll })], - streamMuxers: [mplex()], - connectionEncryption: [noise()], - ...options, - services: { - identify: identify({ - agentVersion: userAgent ?? DefaultUserAgent - }), - ping: ping(), - ...metadataService, - ...pubsubService, - ...options?.services - } - }) as any as Libp2p; // TODO: make libp2p include it; -} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 723105f1d5..0cdf46482b 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -7,7 +7,8 @@ export { export { utf8ToBytes, bytesToUtf8 } from "@waku/utils/bytes"; -export * from "./content_topic.js"; +export { defaultLibp2p } from "./utils/libp2p.js"; +export * from "./utils/content_topic.js"; export * from "./waku.js"; export * from "./create.js"; export * as waku from "@waku/core"; diff --git a/packages/sdk/src/relay/index.ts b/packages/sdk/src/relay/index.ts index 9e90697c6c..6fdc39a32d 100644 --- a/packages/sdk/src/relay/index.ts +++ b/packages/sdk/src/relay/index.ts @@ -1,13 +1,9 @@ -import { - DefaultPubsubTopic, - type ProtocolCreateOptions, - type RelayNode -} from "@waku/interfaces"; -import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay"; -import { ensureShardingConfigured } from "@waku/utils"; +import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core"; +import { type FullNode, type RelayNode } from "@waku/interfaces"; +import { RelayCreateOptions, wakuRelay } from "@waku/relay"; -import { defaultLibp2p, defaultPeerDiscoveries } from "../create.js"; -import { WakuNode, WakuOptions } from "../waku.js"; +import { createLibp2pAndUpdateOptions } from "../utils/libp2p.js"; +import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "../waku.js"; /** * Create a Waku node that uses Waku Relay to send and receive messages, @@ -20,35 +16,13 @@ import { WakuNode, WakuOptions } from "../waku.js"; * or use this function with caution. */ export async function createRelayNode( - options?: ProtocolCreateOptions & - Partial & - Partial -): Promise { - options = options ?? { pubsubTopics: [] }; - - const libp2pOptions = options?.libp2p ?? {}; - const peerDiscovery = libp2pOptions.peerDiscovery ?? []; - - const shardInfo = options.shardInfo - ? ensureShardingConfigured(options.shardInfo) - : undefined; - - options.pubsubTopics = shardInfo?.pubsubTopics ?? - options.pubsubTopics ?? [DefaultPubsubTopic]; - - if (options?.defaultBootstrap) { - peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics)); - Object.assign(libp2pOptions, { peerDiscovery }); + options: CreateWakuNodeOptions & Partial = { + pubsubTopics: [] } +): Promise { + const libp2p = await createLibp2pAndUpdateOptions(options); - const libp2p = await defaultLibp2p( - shardInfo?.shardInfo, - wakuGossipSub(options), - libp2pOptions, - options?.userAgent - ); - - const relay = wakuRelay(options.pubsubTopics); + const relay = wakuRelay(options?.pubsubTopics || []); return new WakuNode( options as WakuOptions, @@ -59,3 +33,38 @@ export async function createRelayNode( relay ) as RelayNode; } + +/** + * Create a Waku node that uses all Waku protocols. + * + * This helper is not recommended except if: + * - you are interfacing with nwaku v0.11 or below + * - you are doing some form of testing + * + * If you are building a full node, it is recommended to use + * [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API. + * + * @see https://github.com/status-im/nwaku/issues/1085 + * @internal + */ +export async function createFullNode( + options: CreateWakuNodeOptions & Partial = { + pubsubTopics: [] + } +): Promise { + const libp2p = await createLibp2pAndUpdateOptions(options); + + const store = wakuStore(options); + const lightPush = wakuLightPush(options); + const filter = wakuFilter(options); + const relay = wakuRelay(options?.pubsubTopics || []); + + return new WakuNode( + options as WakuOptions, + libp2p, + store, + lightPush, + filter, + relay + ) as FullNode; +} diff --git a/packages/sdk/src/content_topic.ts b/packages/sdk/src/utils/content_topic.ts similarity index 98% rename from packages/sdk/src/content_topic.ts rename to packages/sdk/src/utils/content_topic.ts index c4a5c789bf..dc716eb190 100644 --- a/packages/sdk/src/content_topic.ts +++ b/packages/sdk/src/utils/content_topic.ts @@ -12,7 +12,7 @@ import { shardInfoToPubsubTopics } from "@waku/utils"; -import { createLightNode } from "./create.js"; +import { createLightNode } from "../create.js"; interface CreateTopicOptions { waku?: LightNode; diff --git a/packages/sdk/src/utils/discovery.ts b/packages/sdk/src/utils/discovery.ts new file mode 100644 index 0000000000..6fb1ccd3c9 --- /dev/null +++ b/packages/sdk/src/utils/discovery.ts @@ -0,0 +1,22 @@ +import type { PeerDiscovery } from "@libp2p/interface"; +import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery"; +import { type Libp2pComponents, PubsubTopic } from "@waku/interfaces"; +import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery"; +import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange"; + +const DEFAULT_NODE_REQUIREMENTS = { + lightPush: 1, + filter: 1, + store: 1 +}; + +export function defaultPeerDiscoveries( + pubsubTopics: PubsubTopic[] +): ((components: Libp2pComponents) => PeerDiscovery)[] { + const discoveries = [ + wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS), + wakuLocalPeerCacheDiscovery(), + wakuPeerExchangeDiscovery(pubsubTopics) + ]; + return discoveries; +} diff --git a/packages/sdk/src/utils/libp2p.ts b/packages/sdk/src/utils/libp2p.ts new file mode 100644 index 0000000000..4695b6a033 --- /dev/null +++ b/packages/sdk/src/utils/libp2p.ts @@ -0,0 +1,112 @@ +import type { GossipSub } from "@chainsafe/libp2p-gossipsub"; +import { noise } from "@chainsafe/libp2p-noise"; +import { bootstrap } from "@libp2p/bootstrap"; +import { identify } from "@libp2p/identify"; +import { mplex } from "@libp2p/mplex"; +import { ping } from "@libp2p/ping"; +import { webSockets } from "@libp2p/websockets"; +import { all as filterAll } from "@libp2p/websockets/filters"; +import { wakuMetadata } from "@waku/core"; +import { + type CreateLibp2pOptions, + DefaultPubsubTopic, + type IMetadata, + type Libp2p, + type Libp2pComponents, + type ShardInfo +} from "@waku/interfaces"; +import { wakuGossipSub } from "@waku/relay"; +import { ensureShardingConfigured } from "@waku/utils"; +import { createLibp2p } from "libp2p"; + +import { CreateWakuNodeOptions, DefaultUserAgent } from "../waku.js"; + +import { defaultPeerDiscoveries } from "./discovery.js"; + +type PubsubService = { + pubsub?: (components: Libp2pComponents) => GossipSub; +}; + +type MetadataService = { + metadata?: (components: Libp2pComponents) => IMetadata; +}; + +export async function defaultLibp2p( + shardInfo?: ShardInfo, + wakuGossipSub?: PubsubService["pubsub"], + options?: Partial, + userAgent?: string +): Promise { + if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") { + /* eslint-disable no-console */ + console.info( + "%cIgnore WebSocket connection failures", + "background: gray; color: white; font-size: x-large" + ); + console.info( + "%cWaku tries to discover peers and some of them are expected to fail", + "background: gray; color: white; font-size: x-large" + ); + /* eslint-enable no-console */ + } + + const pubsubService: PubsubService = wakuGossipSub + ? { pubsub: wakuGossipSub } + : {}; + + const metadataService: MetadataService = shardInfo + ? { metadata: wakuMetadata(shardInfo) } + : {}; + + return createLibp2p({ + connectionManager: { + minConnections: 1 + }, + transports: [webSockets({ filter: filterAll })], + streamMuxers: [mplex()], + connectionEncryption: [noise()], + ...options, + services: { + identify: identify({ + agentVersion: userAgent ?? DefaultUserAgent + }), + ping: ping(), + ...metadataService, + ...pubsubService, + ...options?.services + } + }) as any as Libp2p; // TODO: make libp2p include it; +} + +export async function createLibp2pAndUpdateOptions( + options: CreateWakuNodeOptions +): Promise { + const shardInfo = options.shardInfo + ? ensureShardingConfigured(options.shardInfo) + : undefined; + + options.pubsubTopics = shardInfo?.pubsubTopics ?? + options.pubsubTopics ?? [DefaultPubsubTopic]; + + const libp2pOptions = options?.libp2p ?? {}; + const peerDiscovery = libp2pOptions.peerDiscovery ?? []; + + if (options?.defaultBootstrap) { + peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics)); + } + + if (options?.bootstrapPeers) { + peerDiscovery.push(bootstrap({ list: options.bootstrapPeers })); + } + + libp2pOptions.peerDiscovery = peerDiscovery; + + const libp2p = await defaultLibp2p( + shardInfo?.shardInfo, + wakuGossipSub(options), + libp2pOptions, + options?.userAgent + ); + + return libp2p; +} diff --git a/packages/sdk/src/waku.ts b/packages/sdk/src/waku.ts index a4649b5be3..7455a8221c 100644 --- a/packages/sdk/src/waku.ts +++ b/packages/sdk/src/waku.ts @@ -11,13 +11,14 @@ import type { IStore, Libp2p, LightNode, + ProtocolCreateOptions, PubsubTopic, Waku } from "@waku/interfaces"; import { Protocols } from "@waku/interfaces"; import { Logger } from "@waku/utils"; -import { subscribeToContentTopic } from "./content_topic.js"; +import { subscribeToContentTopic } from "./utils/content_topic.js"; export const DefaultPingKeepAliveValueSecs = 5 * 60; export const DefaultRelayKeepAliveValueSecs = 5 * 60; @@ -48,6 +49,9 @@ export interface WakuOptions { pubsubTopics: PubsubTopic[]; } +export type CreateWakuNodeOptions = ProtocolCreateOptions & + Partial; + export class WakuNode implements Waku { public libp2p: Libp2p; public relay?: IRelay; diff --git a/packages/tests/src/lib/service_node.ts b/packages/tests/src/lib/service_node.ts index 1dedb161b2..1ff3af85d2 100644 --- a/packages/tests/src/lib/service_node.ts +++ b/packages/tests/src/lib/service_node.ts @@ -201,9 +201,9 @@ export class ServiceNode { return waitForLine(this.logPath, msg, timeout); } - /** Calls nwaku JSON-RPC API `get_waku_v2_admin_v1_peers` to check - * for known peers - * @throws if WakuNode isn't started. + /** + * Calls nwaku REST API "/admin/v1/peers" to check for known peers + * @throws */ async peers(): Promise { this.checkProcess();