From 11819fc7b14e18385d421facaf2af0832cad1da8 Mon Sep 17 00:00:00 2001 From: Danish Arora <35004822+danisharora099@users.noreply.github.com> Date: Tue, 31 Jan 2023 19:47:46 +0530 Subject: [PATCH] feat: DNS discovery as default bootstrap discovery (#1114) * use DNS discovery as default bootstrap discovery * fix: failing CI * fix: typo * introduce tagging, components & explicit tests libp2p wasn't by default tagging peers with dns-discovery as "bootstrap" -- we are manually now tagging peers with "dns-discovery", and then running tests according to that * fix: package installs * fix: typedoc CI * change tag name from dns-discovery to bootstrap * update tag name in test * fix CI * address review * add: prod enrtree and use as default --- package-lock.json | 108 ++++++++++-------- packages/create/package.json | 2 +- packages/create/src/index.ts | 11 +- packages/dns-discovery/src/dns.spec.ts | 30 ++++- packages/dns-discovery/src/index.ts | 71 +++++++++--- packages/peer-exchange/package.json | 2 +- packages/tests/package.json | 14 ++- .../tests/tests/dns-peer-discovery.spec.ts | 57 +++++++-- 8 files changed, 207 insertions(+), 88 deletions(-) diff --git a/package-lock.json b/package-lock.json index ce3d8fbd6e..1daf6bca27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -171,7 +171,7 @@ } }, "node_modules/@babel/core/node_modules/json5": { - "version": "2.2.1", + "version": "2.2.3", "dev": true, "license": "MIT", "bin": { @@ -1840,21 +1840,6 @@ "node": ">=12.0.0" } }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.16.12", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@eslint/eslintrc": { "version": "1.3.3", "dev": true, @@ -2139,6 +2124,7 @@ }, "node_modules/@libp2p/bootstrap": { "version": "5.0.0", + "dev": true, "license": "Apache-2.0 OR MIT", "dependencies": { "@libp2p/interface-peer-discovery": "^1.0.1", @@ -2155,6 +2141,31 @@ "npm": ">=7.0.0" } }, + "node_modules/@libp2p/components": { + "version": "3.1.1", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@libp2p/interface-address-manager": "^2.0.0", + "@libp2p/interface-connection": "^3.0.1", + "@libp2p/interface-connection-manager": "^1.1.0", + "@libp2p/interface-content-routing": "^1.0.2", + "@libp2p/interface-dht": "^1.0.1", + "@libp2p/interface-metrics": "^3.0.0", + "@libp2p/interface-peer-id": "^1.0.4", + "@libp2p/interface-peer-routing": "^1.0.1", + "@libp2p/interface-peer-store": "^1.2.2", + "@libp2p/interface-pubsub": "^3.0.0", + "@libp2p/interface-registrar": "^2.0.3", + "@libp2p/interface-transport": "^2.0.0", + "@libp2p/interfaces": "^3.0.3", + "err-code": "^3.0.1", + "interface-datastore": "^7.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@libp2p/connection": { "version": "4.0.2", "license": "Apache-2.0 OR MIT", @@ -13274,7 +13285,7 @@ "license": "ISC" }, "node_modules/json5": { - "version": "1.0.1", + "version": "1.0.2", "dev": true, "license": "MIT", "dependencies": { @@ -21031,21 +21042,6 @@ "esbuild-windows-arm64": "0.14.39" } }, - "node_modules/playwright-test/node_modules/esbuild-darwin-64": { - "version": "0.14.39", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/playwright-test/node_modules/globby": { "version": "13.1.2", "dev": true, @@ -25090,7 +25086,7 @@ } }, "node_modules/ua-parser-js": { - "version": "0.7.32", + "version": "0.7.33", "dev": true, "funding": [ { @@ -26398,7 +26394,6 @@ "license": "MIT OR Apache-2.0", "dependencies": { "@chainsafe/libp2p-noise": "^10.1.0", - "@libp2p/bootstrap": "^5.0.0", "@libp2p/interface-address-manager": "^2.0.1", "@libp2p/interface-connection": "^3.0.3", "@libp2p/interface-connection-manager": "^1.3.1", @@ -26415,6 +26410,7 @@ "@libp2p/mplex": "^7.0.0", "@libp2p/websockets": "^5.0.0", "@waku/core": "*", + "@waku/dns-discovery": "*", "@waku/peer-exchange": "*", "interface-datastore": "^7.0.1" }, @@ -26712,6 +26708,7 @@ "version": "0.0.1", "license": "MIT OR Apache-2.0", "dependencies": { + "@libp2p/components": "^3.1.1", "@waku/byte-utils": "*", "@waku/core": "*", "@waku/create": "*", @@ -26721,6 +26718,9 @@ "@waku/message-encryption": "*", "@waku/peer-exchange": "*", "app-root-path": "^3.0.0", + "chai": "^4.3.6", + "debug": "^4.3.4", + "mocha": "^9.1.3", "p-timeout": "^6.0.0", "portfinder": "^1.0.28", "tail": "^2.2.0" @@ -26839,7 +26839,7 @@ }, "dependencies": { "json5": { - "version": "2.2.1", + "version": "2.2.3", "dev": true }, "semver": { @@ -27923,11 +27923,6 @@ } } }, - "@esbuild/darwin-x64": { - "version": "0.16.12", - "dev": true, - "optional": true - }, "@eslint/eslintrc": { "version": "1.3.3", "dev": true, @@ -28103,6 +28098,7 @@ }, "@libp2p/bootstrap": { "version": "5.0.0", + "dev": true, "requires": { "@libp2p/interface-peer-discovery": "^1.0.1", "@libp2p/interface-peer-info": "^1.0.3", @@ -28114,6 +28110,26 @@ "@multiformats/multiaddr": "^11.0.0" } }, + "@libp2p/components": { + "version": "3.1.1", + "requires": { + "@libp2p/interface-address-manager": "^2.0.0", + "@libp2p/interface-connection": "^3.0.1", + "@libp2p/interface-connection-manager": "^1.1.0", + "@libp2p/interface-content-routing": "^1.0.2", + "@libp2p/interface-dht": "^1.0.1", + "@libp2p/interface-metrics": "^3.0.0", + "@libp2p/interface-peer-id": "^1.0.4", + "@libp2p/interface-peer-routing": "^1.0.1", + "@libp2p/interface-peer-store": "^1.2.2", + "@libp2p/interface-pubsub": "^3.0.0", + "@libp2p/interface-registrar": "^2.0.3", + "@libp2p/interface-transport": "^2.0.0", + "@libp2p/interfaces": "^3.0.3", + "err-code": "^3.0.1", + "interface-datastore": "^7.0.0" + } + }, "@libp2p/connection": { "version": "4.0.2", "requires": { @@ -29885,7 +29901,6 @@ "version": "file:packages/create", "requires": { "@chainsafe/libp2p-noise": "^10.1.0", - "@libp2p/bootstrap": "^5.0.0", "@libp2p/interface-address-manager": "^2.0.1", "@libp2p/interface-connection": "^3.0.3", "@libp2p/interface-connection-manager": "^1.3.1", @@ -29907,6 +29922,7 @@ "@typescript-eslint/eslint-plugin": "^5.8.1", "@typescript-eslint/parser": "^5.8.1", "@waku/core": "*", + "@waku/dns-discovery": "*", "@waku/interfaces": "*", "@waku/peer-exchange": "*", "cspell": "^6.17.0", @@ -30144,6 +30160,7 @@ "version": "file:packages/tests", "requires": { "@libp2p/bootstrap": "^5.0.0", + "@libp2p/components": "^3.1.1", "@typescript-eslint/eslint-plugin": "^5.8.1", "@typescript-eslint/parser": "^5.8.1", "@waku/byte-utils": "*", @@ -35646,7 +35663,7 @@ "dev": true }, "json5": { - "version": "1.0.1", + "version": "1.0.2", "dev": true, "requires": { "minimist": "^1.2.0" @@ -40597,11 +40614,6 @@ "esbuild-windows-arm64": "0.14.39" } }, - "esbuild-darwin-64": { - "version": "0.14.39", - "dev": true, - "optional": true - }, "globby": { "version": "13.1.2", "dev": true, @@ -43186,7 +43198,7 @@ "dev": true }, "ua-parser-js": { - "version": "0.7.32", + "version": "0.7.33", "dev": true }, "uc.micro": { diff --git a/packages/create/package.json b/packages/create/package.json index cd5fcdd9a8..9ee3bc4f18 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -51,7 +51,6 @@ }, "dependencies": { "@chainsafe/libp2p-noise": "^10.1.0", - "@libp2p/bootstrap": "^5.0.0", "@libp2p/interface-address-manager": "^2.0.1", "@libp2p/interface-connection": "^3.0.3", "@libp2p/interface-connection-manager": "^1.3.1", @@ -68,6 +67,7 @@ "@libp2p/mplex": "^7.0.0", "@libp2p/websockets": "^5.0.0", "@waku/core": "*", + "@waku/dns-discovery": "*", "@waku/peer-exchange": "*", "interface-datastore": "^7.0.1" }, diff --git a/packages/create/src/index.ts b/packages/create/src/index.ts index 158953364c..7e16df48e5 100644 --- a/packages/create/src/index.ts +++ b/packages/create/src/index.ts @@ -1,5 +1,4 @@ import { noise } from "@chainsafe/libp2p-noise"; -import { bootstrap } from "@libp2p/bootstrap"; import type { PeerDiscovery } from "@libp2p/interface-peer-discovery"; import { mplex } from "@libp2p/mplex"; import { webSockets } from "@libp2p/websockets"; @@ -14,7 +13,7 @@ import { wakuStore, } from "@waku/core"; import { DefaultUserAgent } from "@waku/core"; -import { getPredefinedBootstrapNodes } from "@waku/core/lib/predefined_bootstrap_nodes"; +import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery"; import type { FullNode, IRelay, LightNode, RelayNode } from "@waku/interfaces"; import { wakuPeerExchange } from "@waku/peer-exchange"; import type { Libp2p } from "libp2p"; @@ -22,6 +21,12 @@ import { createLibp2p, Libp2pOptions } from "libp2p"; import type { Libp2pComponents } from "./libp2p_components.js"; +const DEFAULT_NODE_REQUIREMENTS = { + lightPush: 1, + filter: 1, + store: 1, +}; + export { Libp2pComponents }; export interface CreateOptions { @@ -165,7 +170,7 @@ export async function createFullNode( export function defaultPeerDiscovery(): ( components: Libp2pComponents ) => PeerDiscovery { - return bootstrap({ list: getPredefinedBootstrapNodes() }); + return wakuDnsDiscovery(enrTree["PROD"], DEFAULT_NODE_REQUIREMENTS); } export async function defaultLibp2p( diff --git a/packages/dns-discovery/src/dns.spec.ts b/packages/dns-discovery/src/dns.spec.ts index 550d46b1b8..2591a5e7f0 100644 --- a/packages/dns-discovery/src/dns.spec.ts +++ b/packages/dns-discovery/src/dns.spec.ts @@ -3,6 +3,8 @@ import { expect } from "chai"; import { DnsClient, DnsNodeDiscovery } from "./dns.js"; import testData from "./testdata.json" assert { type: "json" }; +import { enrTree } from "./index.js"; + const mockData = testData.dns; const host = "nodes.example.org"; @@ -260,9 +262,6 @@ describe("DNS Node Discovery w/ capabilities", () => { }); describe("DNS Node Discovery [live data]", function () { - const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM"; - const fqdn = "test.waku.nodes.status.im"; - const enrTree = `enrtree://${publicKey}@${fqdn}`; const maxQuantity = 3; before(function () { @@ -275,7 +274,30 @@ describe("DNS Node Discovery [live data]", function () { this.timeout(10000); // Google's dns server address. Needs to be set explicitly to run in CI const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); - const peers = await dnsNodeDiscovery.getPeers([enrTree], { + const peers = await dnsNodeDiscovery.getPeers([enrTree.TEST], { + relay: maxQuantity, + store: maxQuantity, + filter: maxQuantity, + lightPush: maxQuantity, + }); + + expect(peers.length).to.eq(maxQuantity); + + const multiaddrs = peers.map((peer) => peer.multiaddrs).flat(); + + const seen: string[] = []; + for (const ma of multiaddrs) { + expect(ma).to.not.be.undefined; + expect(seen).to.not.include(ma!.toString()); + seen.push(ma!.toString()); + } + }); + + it(`should retrieve ${maxQuantity} multiaddrs for prod.waku.nodes.status.im`, async function () { + this.timeout(10000); + // Google's dns server address. Needs to be set explicitly to run in CI + const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); + const peers = await dnsNodeDiscovery.getPeers([enrTree.PROD], { relay: maxQuantity, store: maxQuantity, filter: maxQuantity, diff --git a/packages/dns-discovery/src/index.ts b/packages/dns-discovery/src/index.ts index f077682fe9..44c7c695fb 100644 --- a/packages/dns-discovery/src/index.ts +++ b/packages/dns-discovery/src/index.ts @@ -5,7 +5,7 @@ import type { import { symbol } from "@libp2p/interface-peer-discovery"; import type { PeerInfo } from "@libp2p/interface-peer-info"; import { CustomEvent, EventEmitter } from "@libp2p/interfaces/events"; -import type { IEnr } from "@waku/interfaces"; +import type { IEnr, PeerExchangeComponents } from "@waku/interfaces"; import { multiaddrsToPeerInfo } from "@waku/libp2p-utils"; import debug from "debug"; @@ -15,6 +15,40 @@ export { NodeCapabilityCount }; const log = debug("waku:peer-discovery-dns"); +const enrTree = { + TEST: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@test.waku.nodes.status.im", + PROD: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@prod.waku.nodes.status.im", +}; + +const DEFAULT_BOOTSTRAP_TAG_NAME = "bootstrap"; +const DEFAULT_BOOTSTRAP_TAG_VALUE = 50; +const DEFAULT_BOOTSTRAP_TAG_TTL = 120000; + +export interface Options { + /** + * ENR URL to use for DNS discovery + */ + enrUrl: string; + /** + * Specifies what type of nodes are wanted from the discovery process + */ + wantedNodeCapabilityCount: Partial; + /** + * Tag a bootstrap peer with this name before "discovering" it (default: 'bootstrap') + */ + tagName?: string; + + /** + * The bootstrap peer tag will have this value (default: 50) + */ + tagValue?: number; + + /** + * Cause the bootstrap peer tag to be removed after this number of ms (default: 2 minutes) + */ + tagTTL?: number; +} + /** * Parse options and expose function to return bootstrap peer addresses. */ @@ -24,19 +58,17 @@ export class PeerDiscoveryDns { private readonly nextPeer: () => AsyncGenerator; private _started: boolean; + private _components: PeerExchangeComponents; + private _options: Options; - /** - * @param enrUrl An EIP-1459 ENR Tree URL. For example: - * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev" - * @param wantedNodeCapabilityCount Specifies what node capabilities - * (protocol) must be returned. - */ - constructor( - enrUrl: string, - wantedNodeCapabilityCount: Partial - ) { + constructor(components: PeerExchangeComponents, options: Options) { super(); this._started = false; + this._components = components; + this._options = options; + + const { enrUrl, wantedNodeCapabilityCount } = options; + log("Use following EIP-1459 ENR Tree URL: ", enrUrl); const dns = DnsNodeDiscovery.dnsOverHttp(); @@ -58,7 +90,15 @@ export class PeerDiscoveryDns for await (const peer of this.nextPeer()) { if (!this._started) return; const peerInfos = multiaddrsToPeerInfo(peer.getFullMultiaddrs()); - peerInfos.forEach((peerInfo) => { + peerInfos.forEach(async (peerInfo) => { + await this._components.peerStore.tagPeer( + peerInfo.id, + DEFAULT_BOOTSTRAP_TAG_NAME, + { + value: this._options.tagValue ?? DEFAULT_BOOTSTRAP_TAG_VALUE, + ttl: this._options.tagTTL ?? DEFAULT_BOOTSTRAP_TAG_TTL, + } + ); this.dispatchEvent( new CustomEvent("peer", { detail: peerInfo }) ); @@ -85,8 +125,11 @@ export class PeerDiscoveryDns export function wakuDnsDiscovery( enrUrl: string, wantedNodeCapabilityCount: Partial -): () => PeerDiscoveryDns { - return () => new PeerDiscoveryDns(enrUrl, wantedNodeCapabilityCount); +): (components: PeerExchangeComponents) => PeerDiscoveryDns { + return (components: PeerExchangeComponents) => + new PeerDiscoveryDns(components, { enrUrl, wantedNodeCapabilityCount }); } export { DnsNodeDiscovery, SearchContext, DnsClient } from "./dns.js"; + +export { enrTree }; diff --git a/packages/peer-exchange/package.json b/packages/peer-exchange/package.json index b996a2ebca..3f076c3802 100644 --- a/packages/peer-exchange/package.json +++ b/packages/peer-exchange/package.json @@ -58,8 +58,8 @@ "@waku/core": "*", "@waku/enr": "*", "@waku/interfaces": "*", - "@waku/proto": "*", "@waku/libp2p-utils": "*", + "@waku/proto": "*", "debug": "^4.3.4", "it-all": "^1.0.6", "it-length-prefixed": "^8.0.2", diff --git a/packages/tests/package.json b/packages/tests/package.json index 6ac03a26b9..4321f89dfc 100644 --- a/packages/tests/package.json +++ b/packages/tests/package.json @@ -55,18 +55,22 @@ "node": ">=16" }, "dependencies": { - "app-root-path": "^3.0.0", - "portfinder": "^1.0.28", - "p-timeout": "^6.0.0", - "tail": "^2.2.0", + "@libp2p/components": "^3.1.1", "@waku/byte-utils": "*", "@waku/core": "*", "@waku/create": "*", + "@waku/dns-discovery": "*", "@waku/enr": "*", "@waku/interfaces": "*", "@waku/message-encryption": "*", "@waku/peer-exchange": "*", - "@waku/dns-discovery": "*" + "app-root-path": "^3.0.0", + "chai": "^4.3.6", + "debug": "^4.3.4", + "mocha": "^9.1.3", + "p-timeout": "^6.0.0", + "portfinder": "^1.0.28", + "tail": "^2.2.0" }, "devDependencies": { "mocha": "^9.1.3", diff --git a/packages/tests/tests/dns-peer-discovery.spec.ts b/packages/tests/tests/dns-peer-discovery.spec.ts index 18f3610373..1f9ab6e550 100644 --- a/packages/tests/tests/dns-peer-discovery.spec.ts +++ b/packages/tests/tests/dns-peer-discovery.spec.ts @@ -1,20 +1,38 @@ +import { Components } from "@libp2p/components"; import tests from "@libp2p/interface-peer-discovery-compliance-tests"; +import { Peer } from "@libp2p/interface-peer-store"; +import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; +import { PersistentPeerStore } from "@libp2p/peer-store"; import { createLightNode } from "@waku/create"; -import { DnsNodeDiscovery, wakuDnsDiscovery } from "@waku/dns-discovery"; +import { + DnsNodeDiscovery, + enrTree, + PeerDiscoveryDns, + wakuDnsDiscovery, +} from "@waku/dns-discovery"; import { expect } from "chai"; +import { MemoryDatastore } from "datastore-core"; -const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM"; -const fqdn = "test.waku.nodes.status.im"; -const enrTree = `enrtree://${publicKey}@${fqdn}`; const maxQuantity = 3; describe("DNS Discovery: Compliance Test", async function () { this.timeout(10000); tests({ async setup() { - return wakuDnsDiscovery(enrTree, { - filter: 1, - })(); + // create libp2p mock peerStore + const components = new Components({ + peerStore: new PersistentPeerStore({ + peerId: await createSecp256k1PeerId(), + datastore: new MemoryDatastore(), + }), + }); + + return new PeerDiscoveryDns(components, { + enrUrl: enrTree["PROD"], + wantedNodeCapabilityCount: { + filter: 1, + }, + }); }, async teardown() { // @@ -42,22 +60,37 @@ describe("DNS Node Discovery [live data]", function () { const waku = await createLightNode({ libp2p: { - peerDiscovery: [wakuDnsDiscovery(enrTree, nodeRequirements)], + peerDiscovery: [wakuDnsDiscovery(enrTree["PROD"], nodeRequirements)], }, }); await waku.start(); - const peersFound = await waku.libp2p.peerStore.all(); - expect(peersFound.length).to.eq(maxQuantity); + const allPeers = await waku.libp2p.peerStore.all(); + + const dnsPeers: Peer[] = []; + + for (const peer of allPeers) { + const tags = await waku.libp2p.peerStore.getTags(peer.id); + let hasTag = false; + for (const tag of tags) { + hasTag = tag.name === "bootstrap"; + if (hasTag) { + dnsPeers.push(peer); + break; + } + } + expect(hasTag).to.be.eq(true); + } + expect(dnsPeers.length).to.eq(maxQuantity); }); - it(`should retrieve ${maxQuantity} multiaddrs for prod.nodes.status.im`, async function () { + it(`should retrieve ${maxQuantity} multiaddrs for test.waku.nodes.status.im`, async function () { this.timeout(10000); // Google's dns server address. Needs to be set explicitly to run in CI const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); - const peers = await dnsNodeDiscovery.getPeers([enrTree], { + const peers = await dnsNodeDiscovery.getPeers([enrTree["PROD"]], { relay: maxQuantity, store: maxQuantity, filter: maxQuantity,