feat: DNS discovery as default bootstrap discovery (#1114)

* use DNS discovery as default bootstrap discovery

* fix: failing CI

* fix: typo

* introduce tagging, components & explicit tests

    libp2p wasn't by default tagging peers with dns-discovery as
"bootstrap"
    -- we are manually now tagging peers with "dns-discovery", and then
    running tests according to that

* fix: package installs

* fix: typedoc CI

* change tag name from dns-discovery to bootstrap

* update tag name in test

* fix CI

* address review

* add: prod enrtree and use as default

package-lock.json

@@ -171,7 +171,7 @@
       }
     },
     "node_modules/@babel/core/node_modules/json5": {
-      "version": "2.2.1",
+      "version": "2.2.3",
       "dev": true,
       "license": "MIT",
       "bin": {
@@ -1840,21 +1840,6 @@
         "node": ">=12.0.0"
       }
     },
-    "node_modules/@esbuild/darwin-x64": {
-      "version": "0.16.12",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=12"
-      }
-    },
     "node_modules/@eslint/eslintrc": {
       "version": "1.3.3",
       "dev": true,
@@ -2139,6 +2124,7 @@
     },
     "node_modules/@libp2p/bootstrap": {
       "version": "5.0.0",
+      "dev": true,
       "license": "Apache-2.0 OR MIT",
       "dependencies": {
         "@libp2p/interface-peer-discovery": "^1.0.1",
@@ -2155,6 +2141,31 @@
         "npm": ">=7.0.0"
       }
     },
+    "node_modules/@libp2p/components": {
+      "version": "3.1.1",
+      "license": "Apache-2.0 OR MIT",
+      "dependencies": {
+        "@libp2p/interface-address-manager": "^2.0.0",
+        "@libp2p/interface-connection": "^3.0.1",
+        "@libp2p/interface-connection-manager": "^1.1.0",
+        "@libp2p/interface-content-routing": "^1.0.2",
+        "@libp2p/interface-dht": "^1.0.1",
+        "@libp2p/interface-metrics": "^3.0.0",
+        "@libp2p/interface-peer-id": "^1.0.4",
+        "@libp2p/interface-peer-routing": "^1.0.1",
+        "@libp2p/interface-peer-store": "^1.2.2",
+        "@libp2p/interface-pubsub": "^3.0.0",
+        "@libp2p/interface-registrar": "^2.0.3",
+        "@libp2p/interface-transport": "^2.0.0",
+        "@libp2p/interfaces": "^3.0.3",
+        "err-code": "^3.0.1",
+        "interface-datastore": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=16.0.0",
+        "npm": ">=7.0.0"
+      }
+    },
     "node_modules/@libp2p/connection": {
       "version": "4.0.2",
       "license": "Apache-2.0 OR MIT",
@@ -13274,7 +13285,7 @@
       "license": "ISC"
     },
     "node_modules/json5": {
-      "version": "1.0.1",
+      "version": "1.0.2",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -21031,21 +21042,6 @@
         "esbuild-windows-arm64": "0.14.39"
       }
     },
-    "node_modules/playwright-test/node_modules/esbuild-darwin-64": {
-      "version": "0.14.39",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=12"
-      }
-    },
     "node_modules/playwright-test/node_modules/globby": {
       "version": "13.1.2",
       "dev": true,
@@ -25090,7 +25086,7 @@
       }
     },
     "node_modules/ua-parser-js": {
-      "version": "0.7.32",
+      "version": "0.7.33",
       "dev": true,
       "funding": [
         {
@@ -26398,7 +26394,6 @@
       "license": "MIT OR Apache-2.0",
       "dependencies": {
         "@chainsafe/libp2p-noise": "^10.1.0",
-        "@libp2p/bootstrap": "^5.0.0",
         "@libp2p/interface-address-manager": "^2.0.1",
         "@libp2p/interface-connection": "^3.0.3",
         "@libp2p/interface-connection-manager": "^1.3.1",
@@ -26415,6 +26410,7 @@
         "@libp2p/mplex": "^7.0.0",
         "@libp2p/websockets": "^5.0.0",
         "@waku/core": "*",
+        "@waku/dns-discovery": "*",
         "@waku/peer-exchange": "*",
         "interface-datastore": "^7.0.1"
       },
@@ -26712,6 +26708,7 @@
       "version": "0.0.1",
       "license": "MIT OR Apache-2.0",
       "dependencies": {
+        "@libp2p/components": "^3.1.1",
         "@waku/byte-utils": "*",
         "@waku/core": "*",
         "@waku/create": "*",
@@ -26721,6 +26718,9 @@
         "@waku/message-encryption": "*",
         "@waku/peer-exchange": "*",
         "app-root-path": "^3.0.0",
+        "chai": "^4.3.6",
+        "debug": "^4.3.4",
+        "mocha": "^9.1.3",
         "p-timeout": "^6.0.0",
         "portfinder": "^1.0.28",
         "tail": "^2.2.0"
@@ -26839,7 +26839,7 @@
       },
       "dependencies": {
         "json5": {
-          "version": "2.2.1",
+          "version": "2.2.3",
           "dev": true
         },
         "semver": {
@@ -27923,11 +27923,6 @@
         }
       }
     },
-    "@esbuild/darwin-x64": {
-      "version": "0.16.12",
-      "dev": true,
-      "optional": true
-    },
     "@eslint/eslintrc": {
       "version": "1.3.3",
       "dev": true,
@@ -28103,6 +28098,7 @@
     },
     "@libp2p/bootstrap": {
       "version": "5.0.0",
+      "dev": true,
       "requires": {
         "@libp2p/interface-peer-discovery": "^1.0.1",
         "@libp2p/interface-peer-info": "^1.0.3",
@@ -28114,6 +28110,26 @@
         "@multiformats/multiaddr": "^11.0.0"
       }
     },
+    "@libp2p/components": {
+      "version": "3.1.1",
+      "requires": {
+        "@libp2p/interface-address-manager": "^2.0.0",
+        "@libp2p/interface-connection": "^3.0.1",
+        "@libp2p/interface-connection-manager": "^1.1.0",
+        "@libp2p/interface-content-routing": "^1.0.2",
+        "@libp2p/interface-dht": "^1.0.1",
+        "@libp2p/interface-metrics": "^3.0.0",
+        "@libp2p/interface-peer-id": "^1.0.4",
+        "@libp2p/interface-peer-routing": "^1.0.1",
+        "@libp2p/interface-peer-store": "^1.2.2",
+        "@libp2p/interface-pubsub": "^3.0.0",
+        "@libp2p/interface-registrar": "^2.0.3",
+        "@libp2p/interface-transport": "^2.0.0",
+        "@libp2p/interfaces": "^3.0.3",
+        "err-code": "^3.0.1",
+        "interface-datastore": "^7.0.0"
+      }
+    },
     "@libp2p/connection": {
       "version": "4.0.2",
       "requires": {
@@ -29885,7 +29901,6 @@
       "version": "file:packages/create",
       "requires": {
         "@chainsafe/libp2p-noise": "^10.1.0",
-        "@libp2p/bootstrap": "^5.0.0",
         "@libp2p/interface-address-manager": "^2.0.1",
         "@libp2p/interface-connection": "^3.0.3",
         "@libp2p/interface-connection-manager": "^1.3.1",
@@ -29907,6 +29922,7 @@
         "@typescript-eslint/eslint-plugin": "^5.8.1",
         "@typescript-eslint/parser": "^5.8.1",
         "@waku/core": "*",
+        "@waku/dns-discovery": "*",
         "@waku/interfaces": "*",
         "@waku/peer-exchange": "*",
         "cspell": "^6.17.0",
@@ -30144,6 +30160,7 @@
       "version": "file:packages/tests",
       "requires": {
         "@libp2p/bootstrap": "^5.0.0",
+        "@libp2p/components": "^3.1.1",
         "@typescript-eslint/eslint-plugin": "^5.8.1",
         "@typescript-eslint/parser": "^5.8.1",
         "@waku/byte-utils": "*",
@@ -35646,7 +35663,7 @@
       "dev": true
     },
     "json5": {
-      "version": "1.0.1",
+      "version": "1.0.2",
       "dev": true,
       "requires": {
         "minimist": "^1.2.0"
@@ -40597,11 +40614,6 @@
             "esbuild-windows-arm64": "0.14.39"
           }
         },
-        "esbuild-darwin-64": {
-          "version": "0.14.39",
-          "dev": true,
-          "optional": true
-        },
         "globby": {
           "version": "13.1.2",
           "dev": true,
@@ -43186,7 +43198,7 @@
       "dev": true
     },
     "ua-parser-js": {
-      "version": "0.7.32",
+      "version": "0.7.33",
       "dev": true
     },
     "uc.micro": {

packages/create/package.json

@@ -51,7 +51,6 @@
   },
   "dependencies": {
     "@chainsafe/libp2p-noise": "^10.1.0",
-    "@libp2p/bootstrap": "^5.0.0",
     "@libp2p/interface-address-manager": "^2.0.1",
     "@libp2p/interface-connection": "^3.0.3",
     "@libp2p/interface-connection-manager": "^1.3.1",
@@ -68,6 +67,7 @@
     "@libp2p/mplex": "^7.0.0",
     "@libp2p/websockets": "^5.0.0",
     "@waku/core": "*",
+    "@waku/dns-discovery": "*",
     "@waku/peer-exchange": "*",
     "interface-datastore": "^7.0.1"
   },

packages/create/src/index.ts

@@ -1,5 +1,4 @@
 import { noise } from "@chainsafe/libp2p-noise";
-import { bootstrap } from "@libp2p/bootstrap";
 import type { PeerDiscovery } from "@libp2p/interface-peer-discovery";
 import { mplex } from "@libp2p/mplex";
 import { webSockets } from "@libp2p/websockets";
@@ -14,7 +13,7 @@ import {
   wakuStore,
 } from "@waku/core";
 import { DefaultUserAgent } from "@waku/core";
-import { getPredefinedBootstrapNodes } from "@waku/core/lib/predefined_bootstrap_nodes";
+import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
 import type { FullNode, IRelay, LightNode, RelayNode } from "@waku/interfaces";
 import { wakuPeerExchange } from "@waku/peer-exchange";
 import type { Libp2p } from "libp2p";
@@ -22,6 +21,12 @@ import { createLibp2p, Libp2pOptions } from "libp2p";
 
 import type { Libp2pComponents } from "./libp2p_components.js";
 
+const DEFAULT_NODE_REQUIREMENTS = {
+  lightPush: 1,
+  filter: 1,
+  store: 1,
+};
+
 export { Libp2pComponents };
 
 export interface CreateOptions {
@@ -165,7 +170,7 @@ export async function createFullNode(
 export function defaultPeerDiscovery(): (
   components: Libp2pComponents
 ) => PeerDiscovery {
-  return bootstrap({ list: getPredefinedBootstrapNodes() });
+  return wakuDnsDiscovery(enrTree["PROD"], DEFAULT_NODE_REQUIREMENTS);
 }
 
 export async function defaultLibp2p(

packages/dns-discovery/src/dns.spec.ts

@@ -3,6 +3,8 @@ import { expect } from "chai";
 import { DnsClient, DnsNodeDiscovery } from "./dns.js";
 import testData from "./testdata.json" assert { type: "json" };
 
+import { enrTree } from "./index.js";
+
 const mockData = testData.dns;
 
 const host = "nodes.example.org";
@@ -260,9 +262,6 @@ describe("DNS Node Discovery w/ capabilities", () => {
 });
 
 describe("DNS Node Discovery [live data]", function () {
-  const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM";
-  const fqdn = "test.waku.nodes.status.im";
-  const enrTree = `enrtree://${publicKey}@${fqdn}`;
   const maxQuantity = 3;
 
   before(function () {
@@ -275,7 +274,30 @@ describe("DNS Node Discovery [live data]", function () {
     this.timeout(10000);
     // Google's dns server address. Needs to be set explicitly to run in CI
     const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
-    const peers = await dnsNodeDiscovery.getPeers([enrTree], {
+    const peers = await dnsNodeDiscovery.getPeers([enrTree.TEST], {
+      relay: maxQuantity,
+      store: maxQuantity,
+      filter: maxQuantity,
+      lightPush: maxQuantity,
+    });
+
+    expect(peers.length).to.eq(maxQuantity);
+
+    const multiaddrs = peers.map((peer) => peer.multiaddrs).flat();
+
+    const seen: string[] = [];
+    for (const ma of multiaddrs) {
+      expect(ma).to.not.be.undefined;
+      expect(seen).to.not.include(ma!.toString());
+      seen.push(ma!.toString());
+    }
+  });
+
+  it(`should retrieve ${maxQuantity} multiaddrs for prod.waku.nodes.status.im`, async function () {
+    this.timeout(10000);
+    // Google's dns server address. Needs to be set explicitly to run in CI
+    const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
+    const peers = await dnsNodeDiscovery.getPeers([enrTree.PROD], {
       relay: maxQuantity,
       store: maxQuantity,
       filter: maxQuantity,

packages/dns-discovery/src/index.ts

@@ -5,7 +5,7 @@ import type {
 import { symbol } from "@libp2p/interface-peer-discovery";
 import type { PeerInfo } from "@libp2p/interface-peer-info";
 import { CustomEvent, EventEmitter } from "@libp2p/interfaces/events";
-import type { IEnr } from "@waku/interfaces";
+import type { IEnr, PeerExchangeComponents } from "@waku/interfaces";
 import { multiaddrsToPeerInfo } from "@waku/libp2p-utils";
 import debug from "debug";
 
@@ -15,6 +15,40 @@ export { NodeCapabilityCount };
 
 const log = debug("waku:peer-discovery-dns");
 
+const enrTree = {
+  TEST: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@test.waku.nodes.status.im",
+  PROD: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@prod.waku.nodes.status.im",
+};
+
+const DEFAULT_BOOTSTRAP_TAG_NAME = "bootstrap";
+const DEFAULT_BOOTSTRAP_TAG_VALUE = 50;
+const DEFAULT_BOOTSTRAP_TAG_TTL = 120000;
+
+export interface Options {
+  /**
+   * ENR URL to use for DNS discovery
+   */
+  enrUrl: string;
+  /**
+   * Specifies what type of nodes are wanted from the discovery process
+   */
+  wantedNodeCapabilityCount: Partial<NodeCapabilityCount>;
+  /**
+   * Tag a bootstrap peer with this name before "discovering" it (default: 'bootstrap')
+   */
+  tagName?: string;
+
+  /**
+   * The bootstrap peer tag will have this value (default: 50)
+   */
+  tagValue?: number;
+
+  /**
+   * Cause the bootstrap peer tag to be removed after this number of ms (default: 2 minutes)
+   */
+  tagTTL?: number;
+}
+
 /**
  * Parse options and expose function to return bootstrap peer addresses.
  */
@@ -24,19 +58,17 @@ export class PeerDiscoveryDns
 {
   private readonly nextPeer: () => AsyncGenerator<IEnr>;
   private _started: boolean;
+  private _components: PeerExchangeComponents;
+  private _options: Options;
 
-  /**
+  constructor(components: PeerExchangeComponents, options: Options) {
-   * @param enrUrl An EIP-1459 ENR Tree URL. For example:
-   * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev"
-   * @param wantedNodeCapabilityCount Specifies what node capabilities
-   * (protocol) must be returned.
-   */
-  constructor(
-    enrUrl: string,
-    wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
-  ) {
     super();
     this._started = false;
+    this._components = components;
+    this._options = options;
+
+    const { enrUrl, wantedNodeCapabilityCount } = options;
+
     log("Use following EIP-1459 ENR Tree URL: ", enrUrl);
 
     const dns = DnsNodeDiscovery.dnsOverHttp();
@@ -58,7 +90,15 @@ export class PeerDiscoveryDns
     for await (const peer of this.nextPeer()) {
       if (!this._started) return;
       const peerInfos = multiaddrsToPeerInfo(peer.getFullMultiaddrs());
-      peerInfos.forEach((peerInfo) => {
+      peerInfos.forEach(async (peerInfo) => {
+        await this._components.peerStore.tagPeer(
+          peerInfo.id,
+          DEFAULT_BOOTSTRAP_TAG_NAME,
+          {
+            value: this._options.tagValue ?? DEFAULT_BOOTSTRAP_TAG_VALUE,
+            ttl: this._options.tagTTL ?? DEFAULT_BOOTSTRAP_TAG_TTL,
+          }
+        );
         this.dispatchEvent(
           new CustomEvent<PeerInfo>("peer", { detail: peerInfo })
         );
@@ -85,8 +125,11 @@ export class PeerDiscoveryDns
 export function wakuDnsDiscovery(
   enrUrl: string,
   wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
-): () => PeerDiscoveryDns {
+): (components: PeerExchangeComponents) => PeerDiscoveryDns {
-  return () => new PeerDiscoveryDns(enrUrl, wantedNodeCapabilityCount);
+  return (components: PeerExchangeComponents) =>
+    new PeerDiscoveryDns(components, { enrUrl, wantedNodeCapabilityCount });
 }
 
 export { DnsNodeDiscovery, SearchContext, DnsClient } from "./dns.js";
+
+export { enrTree };

packages/peer-exchange/package.json

@@ -58,8 +58,8 @@
     "@waku/core": "*",
     "@waku/enr": "*",
     "@waku/interfaces": "*",
-    "@waku/proto": "*",
     "@waku/libp2p-utils": "*",
+    "@waku/proto": "*",
     "debug": "^4.3.4",
     "it-all": "^1.0.6",
     "it-length-prefixed": "^8.0.2",

packages/tests/package.json

@@ -55,18 +55,22 @@
     "node": ">=16"
   },
   "dependencies": {
-    "app-root-path": "^3.0.0",
+    "@libp2p/components": "^3.1.1",
-    "portfinder": "^1.0.28",
-    "p-timeout": "^6.0.0",
-    "tail": "^2.2.0",
     "@waku/byte-utils": "*",
     "@waku/core": "*",
     "@waku/create": "*",
+    "@waku/dns-discovery": "*",
     "@waku/enr": "*",
     "@waku/interfaces": "*",
     "@waku/message-encryption": "*",
     "@waku/peer-exchange": "*",
-    "@waku/dns-discovery": "*"
+    "app-root-path": "^3.0.0",
+    "chai": "^4.3.6",
+    "debug": "^4.3.4",
+    "mocha": "^9.1.3",
+    "p-timeout": "^6.0.0",
+    "portfinder": "^1.0.28",
+    "tail": "^2.2.0"
   },
   "devDependencies": {
     "mocha": "^9.1.3",

packages/tests/tests/dns-peer-discovery.spec.ts

@@ -1,20 +1,38 @@
+import { Components } from "@libp2p/components";
 import tests from "@libp2p/interface-peer-discovery-compliance-tests";
+import { Peer } from "@libp2p/interface-peer-store";
+import { createSecp256k1PeerId } from "@libp2p/peer-id-factory";
+import { PersistentPeerStore } from "@libp2p/peer-store";
 import { createLightNode } from "@waku/create";
-import { DnsNodeDiscovery, wakuDnsDiscovery } from "@waku/dns-discovery";
+import {
+  DnsNodeDiscovery,
+  enrTree,
+  PeerDiscoveryDns,
+  wakuDnsDiscovery,
+} from "@waku/dns-discovery";
 import { expect } from "chai";
+import { MemoryDatastore } from "datastore-core";
 
-const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM";
-const fqdn = "test.waku.nodes.status.im";
-const enrTree = `enrtree://${publicKey}@${fqdn}`;
 const maxQuantity = 3;
 
 describe("DNS Discovery: Compliance Test", async function () {
   this.timeout(10000);
   tests({
     async setup() {
-      return wakuDnsDiscovery(enrTree, {
+      // create libp2p mock peerStore
-        filter: 1,
+      const components = new Components({
-      })();
+        peerStore: new PersistentPeerStore({
+          peerId: await createSecp256k1PeerId(),
+          datastore: new MemoryDatastore(),
+        }),
+      });
+
+      return new PeerDiscoveryDns(components, {
+        enrUrl: enrTree["PROD"],
+        wantedNodeCapabilityCount: {
+          filter: 1,
+        },
+      });
     },
     async teardown() {
       //
@@ -42,22 +60,37 @@ describe("DNS Node Discovery [live data]", function () {
 
     const waku = await createLightNode({
       libp2p: {
-        peerDiscovery: [wakuDnsDiscovery(enrTree, nodeRequirements)],
+        peerDiscovery: [wakuDnsDiscovery(enrTree["PROD"], nodeRequirements)],
       },
     });
 
     await waku.start();
 
-    const peersFound = await waku.libp2p.peerStore.all();
+    const allPeers = await waku.libp2p.peerStore.all();
-    expect(peersFound.length).to.eq(maxQuantity);
+
+    const dnsPeers: Peer[] = [];
+
+    for (const peer of allPeers) {
+      const tags = await waku.libp2p.peerStore.getTags(peer.id);
+      let hasTag = false;
+      for (const tag of tags) {
+        hasTag = tag.name === "bootstrap";
+        if (hasTag) {
+          dnsPeers.push(peer);
+          break;
+        }
+      }
+      expect(hasTag).to.be.eq(true);
+    }
+    expect(dnsPeers.length).to.eq(maxQuantity);
   });
 
-  it(`should retrieve ${maxQuantity} multiaddrs for prod.nodes.status.im`, async function () {
+  it(`should retrieve ${maxQuantity} multiaddrs for test.waku.nodes.status.im`, async function () {
     this.timeout(10000);
     // Google's dns server address. Needs to be set explicitly to run in CI
     const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
 
-    const peers = await dnsNodeDiscovery.getPeers([enrTree], {
+    const peers = await dnsNodeDiscovery.getPeers([enrTree["PROD"]], {
       relay: maxQuantity,
       store: maxQuantity,
       filter: maxQuantity,