feat: add bootstrapPeers option and refactor sdk (#1871)

* move relay related code

* move libp2p to utils

* define CreateWakuNodeOptions

* improve options

* make libp2p use from create function

* add bootstrapPeers option

* fix lint

* fix types, imports

* fix exports

* use bootstrap along default bootstrap

* fix test as REST does not return peer though connection is made

* rollback condition change

* enable gossipSub back for every node

package-lock.json

@@ -2714,7 +2714,6 @@
       "version": "10.0.11",
       "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz",
       "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==",
-      "dev": true,
       "dependencies": {
         "@libp2p/interface": "^1.1.1",
         "@libp2p/peer-id": "^4.0.4",
@@ -28811,6 +28810,7 @@
       "license": "MIT OR Apache-2.0",
       "dependencies": {
         "@chainsafe/libp2p-noise": "^14.1.0",
+        "@libp2p/bootstrap": "^10.0.11",
         "@libp2p/identify": "^1.0.11",
         "@libp2p/mplex": "^10.0.12",
         "@libp2p/ping": "^1.0.11",
@@ -28837,6 +28837,9 @@
       },
       "engines": {
         "node": ">=18"
+      },
+      "peerDependencies": {
+        "@libp2p/bootstrap": "^10"
       }
     },
     "packages/tests": {
@@ -30748,7 +30751,6 @@
       "version": "10.0.11",
       "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz",
       "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==",
-      "dev": true,
       "requires": {
         "@libp2p/interface": "^1.1.1",
         "@libp2p/peer-id": "^4.0.4",
@@ -33278,6 +33280,7 @@
       "requires": {
         "@chainsafe/libp2p-gossipsub": "^12.0.0",
         "@chainsafe/libp2p-noise": "^14.1.0",
+        "@libp2p/bootstrap": "^10.0.11",
         "@libp2p/identify": "^1.0.11",
         "@libp2p/mplex": "^10.0.12",
         "@libp2p/ping": "^1.0.11",

packages/interfaces/src/protocols.ts

@@ -87,6 +87,10 @@ export type ProtocolCreateOptions = {
    * Use recommended bootstrap method to discovery and connect to new nodes.
    */
   defaultBootstrap?: boolean;
+  /**
+   * List of peers to use to bootstrap the node. Ignored if defaultBootstrap is set to true.
+   */
+  bootstrapPeers?: string[];
 };
 
 export type Callback<T extends IDecodedMessage> = (

packages/sdk/package.json

@@ -73,7 +73,8 @@
     "@waku/peer-exchange": "^0.0.19",
     "@waku/relay": "0.0.9",
     "@waku/utils": "0.0.14",
-    "libp2p": "^1.1.2"
+    "libp2p": "^1.1.2",
+    "@libp2p/bootstrap": "^10.0.11"
   },
   "devDependencies": {
     "@chainsafe/libp2p-gossipsub": "^12.0.0",
@@ -86,6 +87,9 @@
     "npm-run-all": "^4.1.5",
     "rollup": "^4.12.0"
   },
+  "peerDependencies": {
+    "@libp2p/bootstrap": "^10"
+  },
   "files": [
     "dist",
     "bundle",

packages/sdk/src/create.ts

@@ -1,38 +1,8 @@
-import type { GossipSub } from "@chainsafe/libp2p-gossipsub";
+import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core";
-import { noise } from "@chainsafe/libp2p-noise";
+import { type Libp2pComponents, type LightNode } from "@waku/interfaces";
-import { identify } from "@libp2p/identify";
-import type { PeerDiscovery } from "@libp2p/interface";
-import { mplex } from "@libp2p/mplex";
-import { ping } from "@libp2p/ping";
-import { webSockets } from "@libp2p/websockets";
-import { all as filterAll } from "@libp2p/websockets/filters";
-import { wakuFilter, wakuLightPush, wakuMetadata, wakuStore } from "@waku/core";
-import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
-import {
-  type CreateLibp2pOptions,
-  DefaultPubsubTopic,
-  type FullNode,
-  type IMetadata,
-  type Libp2p,
-  type Libp2pComponents,
-  type LightNode,
-  type ProtocolCreateOptions,
-  PubsubTopic,
-  type ShardInfo
-} from "@waku/interfaces";
-import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery";
-import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange";
-import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
-import { ensureShardingConfigured } from "@waku/utils";
-import { createLibp2p } from "libp2p";
 
-import { DefaultUserAgent, WakuNode, WakuOptions } from "./waku.js";
+import { createLibp2pAndUpdateOptions } from "./utils/libp2p.js";
-
+import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "./waku.js";
-const DEFAULT_NODE_REQUIREMENTS = {
-  lightPush: 1,
-  filter: 1,
-  store: 1
-};
 
 export { Libp2pComponents };
 
@@ -40,33 +10,13 @@ export { Libp2pComponents };
  * Create a Waku node configured to use autosharding or static sharding.
  */
 export async function createNode(
-  options?: ProtocolCreateOptions &
+  options: CreateWakuNodeOptions = { pubsubTopics: [] }
-    Partial<WakuOptions> &
-    Partial<RelayCreateOptions>
 ): Promise<LightNode> {
-  options = options ?? { pubsubTopics: [] };
-
   if (!options.shardInfo) {
     throw new Error("Shard info must be set");
   }
 
-  const shardInfo = ensureShardingConfigured(options.shardInfo);
+  const libp2p = await createLibp2pAndUpdateOptions(options);
-  options.pubsubTopics = shardInfo.pubsubTopics;
-  options.shardInfo = shardInfo.shardInfo;
-
-  const libp2pOptions = options?.libp2p ?? {};
-  const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
-  if (options?.defaultBootstrap) {
-    peerDiscovery.push(...defaultPeerDiscoveries(shardInfo.pubsubTopics));
-    Object.assign(libp2pOptions, { peerDiscovery });
-  }
-
-  const libp2p = await defaultLibp2p(
-    shardInfo.shardInfo,
-    wakuGossipSub(options),
-    libp2pOptions,
-    options?.userAgent
-  );
 
   const store = wakuStore(options);
   const lightPush = wakuLightPush(options);
@@ -87,30 +37,9 @@ export async function createNode(
  * Uses Waku Filter V2 by default.
  */
 export async function createLightNode(
-  options?: ProtocolCreateOptions & Partial<WakuOptions>
+  options: CreateWakuNodeOptions = {}
 ): Promise<LightNode> {
-  options = options ?? {};
+  const libp2p = await createLibp2pAndUpdateOptions(options);
-
-  const shardInfo = options.shardInfo
-    ? ensureShardingConfigured(options.shardInfo)
-    : undefined;
-
-  options.pubsubTopics = shardInfo?.pubsubTopics ??
-    options.pubsubTopics ?? [DefaultPubsubTopic];
-
-  const libp2pOptions = options?.libp2p ?? {};
-  const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
-  if (options?.defaultBootstrap) {
-    peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
-    Object.assign(libp2pOptions, { peerDiscovery });
-  }
-
-  const libp2p = await defaultLibp2p(
-    shardInfo?.shardInfo,
-    wakuGossipSub(options),
-    libp2pOptions,
-    options?.userAgent
-  );
 
   const store = wakuStore(options);
   const lightPush = wakuLightPush(options);
@@ -124,127 +53,3 @@ export async function createLightNode(
     filter
   ) as LightNode;
 }
-
-/**
- * Create a Waku node that uses all Waku protocols.
- *
- * This helper is not recommended except if:
- * - you are interfacing with nwaku v0.11 or below
- * - you are doing some form of testing
- *
- * If you are building a full node, it is recommended to use
- * [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API.
- *
- * @see https://github.com/status-im/nwaku/issues/1085
- * @internal
- */
-export async function createFullNode(
-  options?: ProtocolCreateOptions &
-    Partial<WakuOptions> &
-    Partial<RelayCreateOptions>
-): Promise<FullNode> {
-  options = options ?? { pubsubTopics: [] };
-
-  const shardInfo = options.shardInfo
-    ? ensureShardingConfigured(options.shardInfo)
-    : undefined;
-
-  const pubsubTopics = shardInfo?.pubsubTopics ??
-    options.pubsubTopics ?? [DefaultPubsubTopic];
-  options.pubsubTopics = pubsubTopics;
-  options.shardInfo = shardInfo?.shardInfo;
-
-  const libp2pOptions = options?.libp2p ?? {};
-  const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
-  if (options?.defaultBootstrap) {
-    peerDiscovery.push(...defaultPeerDiscoveries(pubsubTopics));
-    Object.assign(libp2pOptions, { peerDiscovery });
-  }
-
-  const libp2p = await defaultLibp2p(
-    shardInfo?.shardInfo,
-    wakuGossipSub(options),
-    libp2pOptions,
-    options?.userAgent
-  );
-
-  const store = wakuStore(options);
-  const lightPush = wakuLightPush(options);
-  const filter = wakuFilter(options);
-  const relay = wakuRelay(pubsubTopics);
-
-  return new WakuNode(
-    options as WakuOptions,
-    libp2p,
-    store,
-    lightPush,
-    filter,
-    relay
-  ) as FullNode;
-}
-
-export function defaultPeerDiscoveries(
-  pubsubTopics: PubsubTopic[]
-): ((components: Libp2pComponents) => PeerDiscovery)[] {
-  const discoveries = [
-    wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS),
-    wakuLocalPeerCacheDiscovery(),
-    wakuPeerExchangeDiscovery(pubsubTopics)
-  ];
-  return discoveries;
-}
-
-type PubsubService = {
-  pubsub?: (components: Libp2pComponents) => GossipSub;
-};
-
-type MetadataService = {
-  metadata?: (components: Libp2pComponents) => IMetadata;
-};
-
-export async function defaultLibp2p(
-  shardInfo?: ShardInfo,
-  wakuGossipSub?: PubsubService["pubsub"],
-  options?: Partial<CreateLibp2pOptions>,
-  userAgent?: string
-): Promise<Libp2p> {
-  if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") {
-    /* eslint-disable no-console */
-    console.info(
-      "%cIgnore WebSocket connection failures",
-      "background: gray; color: white; font-size: x-large"
-    );
-    console.info(
-      "%cWaku tries to discover peers and some of them are expected to fail",
-      "background: gray; color: white; font-size: x-large"
-    );
-    /* eslint-enable no-console */
-  }
-
-  const pubsubService: PubsubService = wakuGossipSub
-    ? { pubsub: wakuGossipSub }
-    : {};
-
-  const metadataService: MetadataService = shardInfo
-    ? { metadata: wakuMetadata(shardInfo) }
-    : {};
-
-  return createLibp2p({
-    connectionManager: {
-      minConnections: 1
-    },
-    transports: [webSockets({ filter: filterAll })],
-    streamMuxers: [mplex()],
-    connectionEncryption: [noise()],
-    ...options,
-    services: {
-      identify: identify({
-        agentVersion: userAgent ?? DefaultUserAgent
-      }),
-      ping: ping(),
-      ...metadataService,
-      ...pubsubService,
-      ...options?.services
-    }
-  }) as any as Libp2p; // TODO: make libp2p include it;
-}

packages/sdk/src/index.ts

@@ -7,7 +7,8 @@ export {
 
 export { utf8ToBytes, bytesToUtf8 } from "@waku/utils/bytes";
 
-export * from "./content_topic.js";
+export { defaultLibp2p } from "./utils/libp2p.js";
+export * from "./utils/content_topic.js";
 export * from "./waku.js";
 export * from "./create.js";
 export * as waku from "@waku/core";

packages/sdk/src/relay/index.ts

@@ -1,13 +1,9 @@
-import {
+import { wakuFilter, wakuLightPush, wakuStore } from "@waku/core";
-  DefaultPubsubTopic,
+import { type FullNode, type RelayNode } from "@waku/interfaces";
-  type ProtocolCreateOptions,
+import { RelayCreateOptions, wakuRelay } from "@waku/relay";
-  type RelayNode
-} from "@waku/interfaces";
-import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
-import { ensureShardingConfigured } from "@waku/utils";
 
-import { defaultLibp2p, defaultPeerDiscoveries } from "../create.js";
+import { createLibp2pAndUpdateOptions } from "../utils/libp2p.js";
-import { WakuNode, WakuOptions } from "../waku.js";
+import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "../waku.js";
 
 /**
  * Create a Waku node that uses Waku Relay to send and receive messages,
@@ -20,35 +16,13 @@ import { WakuNode, WakuOptions } from "../waku.js";
  * or use this function with caution.
  */
 export async function createRelayNode(
-  options?: ProtocolCreateOptions &
+  options: CreateWakuNodeOptions & Partial<RelayCreateOptions> = {
-    Partial<WakuOptions> &
+    pubsubTopics: []
-    Partial<RelayCreateOptions>
-): Promise<RelayNode> {
-  options = options ?? { pubsubTopics: [] };
-
-  const libp2pOptions = options?.libp2p ?? {};
-  const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
-
-  const shardInfo = options.shardInfo
-    ? ensureShardingConfigured(options.shardInfo)
-    : undefined;
-
-  options.pubsubTopics = shardInfo?.pubsubTopics ??
-    options.pubsubTopics ?? [DefaultPubsubTopic];
-
-  if (options?.defaultBootstrap) {
-    peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
-    Object.assign(libp2pOptions, { peerDiscovery });
   }
+): Promise<RelayNode> {
+  const libp2p = await createLibp2pAndUpdateOptions(options);
 
-  const libp2p = await defaultLibp2p(
+  const relay = wakuRelay(options?.pubsubTopics || []);
-    shardInfo?.shardInfo,
-    wakuGossipSub(options),
-    libp2pOptions,
-    options?.userAgent
-  );
-
-  const relay = wakuRelay(options.pubsubTopics);
 
   return new WakuNode(
     options as WakuOptions,
@@ -59,3 +33,38 @@ export async function createRelayNode(
     relay
   ) as RelayNode;
 }
+
+/**
+ * Create a Waku node that uses all Waku protocols.
+ *
+ * This helper is not recommended except if:
+ * - you are interfacing with nwaku v0.11 or below
+ * - you are doing some form of testing
+ *
+ * If you are building a full node, it is recommended to use
+ * [nwaku](github.com/status-im/nwaku) and its JSON RPC API or wip REST API.
+ *
+ * @see https://github.com/status-im/nwaku/issues/1085
+ * @internal
+ */
+export async function createFullNode(
+  options: CreateWakuNodeOptions & Partial<RelayCreateOptions> = {
+    pubsubTopics: []
+  }
+): Promise<FullNode> {
+  const libp2p = await createLibp2pAndUpdateOptions(options);
+
+  const store = wakuStore(options);
+  const lightPush = wakuLightPush(options);
+  const filter = wakuFilter(options);
+  const relay = wakuRelay(options?.pubsubTopics || []);
+
+  return new WakuNode(
+    options as WakuOptions,
+    libp2p,
+    store,
+    lightPush,
+    filter,
+    relay
+  ) as FullNode;
+}

packages/sdk/src/utils/content_topic.ts

@@ -12,7 +12,7 @@ import {
   shardInfoToPubsubTopics
 } from "@waku/utils";
 
-import { createLightNode } from "./create.js";
+import { createLightNode } from "../create.js";
 
 interface CreateTopicOptions {
   waku?: LightNode;

packages/sdk/src/utils/discovery.ts

@@ -0,0 +1,22 @@
+import type { PeerDiscovery } from "@libp2p/interface";
+import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
+import { type Libp2pComponents, PubsubTopic } from "@waku/interfaces";
+import { wakuLocalPeerCacheDiscovery } from "@waku/local-peer-cache-discovery";
+import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange";
+
+const DEFAULT_NODE_REQUIREMENTS = {
+  lightPush: 1,
+  filter: 1,
+  store: 1
+};
+
+export function defaultPeerDiscoveries(
+  pubsubTopics: PubsubTopic[]
+): ((components: Libp2pComponents) => PeerDiscovery)[] {
+  const discoveries = [
+    wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS),
+    wakuLocalPeerCacheDiscovery(),
+    wakuPeerExchangeDiscovery(pubsubTopics)
+  ];
+  return discoveries;
+}

packages/sdk/src/utils/libp2p.ts

@@ -0,0 +1,112 @@
+import type { GossipSub } from "@chainsafe/libp2p-gossipsub";
+import { noise } from "@chainsafe/libp2p-noise";
+import { bootstrap } from "@libp2p/bootstrap";
+import { identify } from "@libp2p/identify";
+import { mplex } from "@libp2p/mplex";
+import { ping } from "@libp2p/ping";
+import { webSockets } from "@libp2p/websockets";
+import { all as filterAll } from "@libp2p/websockets/filters";
+import { wakuMetadata } from "@waku/core";
+import {
+  type CreateLibp2pOptions,
+  DefaultPubsubTopic,
+  type IMetadata,
+  type Libp2p,
+  type Libp2pComponents,
+  type ShardInfo
+} from "@waku/interfaces";
+import { wakuGossipSub } from "@waku/relay";
+import { ensureShardingConfigured } from "@waku/utils";
+import { createLibp2p } from "libp2p";
+
+import { CreateWakuNodeOptions, DefaultUserAgent } from "../waku.js";
+
+import { defaultPeerDiscoveries } from "./discovery.js";
+
+type PubsubService = {
+  pubsub?: (components: Libp2pComponents) => GossipSub;
+};
+
+type MetadataService = {
+  metadata?: (components: Libp2pComponents) => IMetadata;
+};
+
+export async function defaultLibp2p(
+  shardInfo?: ShardInfo,
+  wakuGossipSub?: PubsubService["pubsub"],
+  options?: Partial<CreateLibp2pOptions>,
+  userAgent?: string
+): Promise<Libp2p> {
+  if (!options?.hideWebSocketInfo && process.env.NODE_ENV !== "test") {
+    /* eslint-disable no-console */
+    console.info(
+      "%cIgnore WebSocket connection failures",
+      "background: gray; color: white; font-size: x-large"
+    );
+    console.info(
+      "%cWaku tries to discover peers and some of them are expected to fail",
+      "background: gray; color: white; font-size: x-large"
+    );
+    /* eslint-enable no-console */
+  }
+
+  const pubsubService: PubsubService = wakuGossipSub
+    ? { pubsub: wakuGossipSub }
+    : {};
+
+  const metadataService: MetadataService = shardInfo
+    ? { metadata: wakuMetadata(shardInfo) }
+    : {};
+
+  return createLibp2p({
+    connectionManager: {
+      minConnections: 1
+    },
+    transports: [webSockets({ filter: filterAll })],
+    streamMuxers: [mplex()],
+    connectionEncryption: [noise()],
+    ...options,
+    services: {
+      identify: identify({
+        agentVersion: userAgent ?? DefaultUserAgent
+      }),
+      ping: ping(),
+      ...metadataService,
+      ...pubsubService,
+      ...options?.services
+    }
+  }) as any as Libp2p; // TODO: make libp2p include it;
+}
+
+export async function createLibp2pAndUpdateOptions(
+  options: CreateWakuNodeOptions
+): Promise<Libp2p> {
+  const shardInfo = options.shardInfo
+    ? ensureShardingConfigured(options.shardInfo)
+    : undefined;
+
+  options.pubsubTopics = shardInfo?.pubsubTopics ??
+    options.pubsubTopics ?? [DefaultPubsubTopic];
+
+  const libp2pOptions = options?.libp2p ?? {};
+  const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
+
+  if (options?.defaultBootstrap) {
+    peerDiscovery.push(...defaultPeerDiscoveries(options.pubsubTopics));
+  }
+
+  if (options?.bootstrapPeers) {
+    peerDiscovery.push(bootstrap({ list: options.bootstrapPeers }));
+  }
+
+  libp2pOptions.peerDiscovery = peerDiscovery;
+
+  const libp2p = await defaultLibp2p(
+    shardInfo?.shardInfo,
+    wakuGossipSub(options),
+    libp2pOptions,
+    options?.userAgent
+  );
+
+  return libp2p;
+}

packages/sdk/src/waku.ts

@@ -11,13 +11,14 @@ import type {
   IStore,
   Libp2p,
   LightNode,
+  ProtocolCreateOptions,
   PubsubTopic,
   Waku
 } from "@waku/interfaces";
 import { Protocols } from "@waku/interfaces";
 import { Logger } from "@waku/utils";
 
-import { subscribeToContentTopic } from "./content_topic.js";
+import { subscribeToContentTopic } from "./utils/content_topic.js";
 
 export const DefaultPingKeepAliveValueSecs = 5 * 60;
 export const DefaultRelayKeepAliveValueSecs = 5 * 60;
@@ -48,6 +49,9 @@ export interface WakuOptions {
   pubsubTopics: PubsubTopic[];
 }
 
+export type CreateWakuNodeOptions = ProtocolCreateOptions &
+  Partial<WakuOptions>;
+
 export class WakuNode implements Waku {
   public libp2p: Libp2p;
   public relay?: IRelay;

packages/tests/src/lib/service_node.ts

@@ -201,9 +201,9 @@ export class ServiceNode {
     return waitForLine(this.logPath, msg, timeout);
   }
 
-  /** Calls nwaku JSON-RPC API `get_waku_v2_admin_v1_peers` to check
+  /**
-   * for known peers
+   * Calls nwaku REST API "/admin/v1/peers" to check for known peers
-   * @throws if WakuNode isn't started.
+   * @throws
    */
   async peers(): Promise<string[]> {
     this.checkProcess();