feat: DNS discovery as default bootstrap discovery (#1114)

* use DNS discovery as default bootstrap discovery

* fix: failing CI

* fix: typo

* introduce tagging, components & explicit tests

    libp2p wasn't by default tagging peers with dns-discovery as
"bootstrap"
    -- we are manually now tagging peers with "dns-discovery", and then
    running tests according to that

* fix: package installs

* fix: typedoc CI

* change tag name from dns-discovery to bootstrap

* update tag name in test

* fix CI

* address review

* add: prod enrtree and use as default
This commit is contained in:
Danish Arora 2023-01-31 19:47:46 +05:30 committed by GitHub
parent 1166dbc51e
commit 11819fc7b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 207 additions and 88 deletions

108
package-lock.json generated
View File

@ -171,7 +171,7 @@
}
},
"node_modules/@babel/core/node_modules/json5": {
"version": "2.2.1",
"version": "2.2.3",
"dev": true,
"license": "MIT",
"bin": {
@ -1840,21 +1840,6 @@
"node": ">=12.0.0"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.16.12",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@eslint/eslintrc": {
"version": "1.3.3",
"dev": true,
@ -2139,6 +2124,7 @@
},
"node_modules/@libp2p/bootstrap": {
"version": "5.0.0",
"dev": true,
"license": "Apache-2.0 OR MIT",
"dependencies": {
"@libp2p/interface-peer-discovery": "^1.0.1",
@ -2155,6 +2141,31 @@
"npm": ">=7.0.0"
}
},
"node_modules/@libp2p/components": {
"version": "3.1.1",
"license": "Apache-2.0 OR MIT",
"dependencies": {
"@libp2p/interface-address-manager": "^2.0.0",
"@libp2p/interface-connection": "^3.0.1",
"@libp2p/interface-connection-manager": "^1.1.0",
"@libp2p/interface-content-routing": "^1.0.2",
"@libp2p/interface-dht": "^1.0.1",
"@libp2p/interface-metrics": "^3.0.0",
"@libp2p/interface-peer-id": "^1.0.4",
"@libp2p/interface-peer-routing": "^1.0.1",
"@libp2p/interface-peer-store": "^1.2.2",
"@libp2p/interface-pubsub": "^3.0.0",
"@libp2p/interface-registrar": "^2.0.3",
"@libp2p/interface-transport": "^2.0.0",
"@libp2p/interfaces": "^3.0.3",
"err-code": "^3.0.1",
"interface-datastore": "^7.0.0"
},
"engines": {
"node": ">=16.0.0",
"npm": ">=7.0.0"
}
},
"node_modules/@libp2p/connection": {
"version": "4.0.2",
"license": "Apache-2.0 OR MIT",
@ -13274,7 +13285,7 @@
"license": "ISC"
},
"node_modules/json5": {
"version": "1.0.1",
"version": "1.0.2",
"dev": true,
"license": "MIT",
"dependencies": {
@ -21031,21 +21042,6 @@
"esbuild-windows-arm64": "0.14.39"
}
},
"node_modules/playwright-test/node_modules/esbuild-darwin-64": {
"version": "0.14.39",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/playwright-test/node_modules/globby": {
"version": "13.1.2",
"dev": true,
@ -25090,7 +25086,7 @@
}
},
"node_modules/ua-parser-js": {
"version": "0.7.32",
"version": "0.7.33",
"dev": true,
"funding": [
{
@ -26398,7 +26394,6 @@
"license": "MIT OR Apache-2.0",
"dependencies": {
"@chainsafe/libp2p-noise": "^10.1.0",
"@libp2p/bootstrap": "^5.0.0",
"@libp2p/interface-address-manager": "^2.0.1",
"@libp2p/interface-connection": "^3.0.3",
"@libp2p/interface-connection-manager": "^1.3.1",
@ -26415,6 +26410,7 @@
"@libp2p/mplex": "^7.0.0",
"@libp2p/websockets": "^5.0.0",
"@waku/core": "*",
"@waku/dns-discovery": "*",
"@waku/peer-exchange": "*",
"interface-datastore": "^7.0.1"
},
@ -26712,6 +26708,7 @@
"version": "0.0.1",
"license": "MIT OR Apache-2.0",
"dependencies": {
"@libp2p/components": "^3.1.1",
"@waku/byte-utils": "*",
"@waku/core": "*",
"@waku/create": "*",
@ -26721,6 +26718,9 @@
"@waku/message-encryption": "*",
"@waku/peer-exchange": "*",
"app-root-path": "^3.0.0",
"chai": "^4.3.6",
"debug": "^4.3.4",
"mocha": "^9.1.3",
"p-timeout": "^6.0.0",
"portfinder": "^1.0.28",
"tail": "^2.2.0"
@ -26839,7 +26839,7 @@
},
"dependencies": {
"json5": {
"version": "2.2.1",
"version": "2.2.3",
"dev": true
},
"semver": {
@ -27923,11 +27923,6 @@
}
}
},
"@esbuild/darwin-x64": {
"version": "0.16.12",
"dev": true,
"optional": true
},
"@eslint/eslintrc": {
"version": "1.3.3",
"dev": true,
@ -28103,6 +28098,7 @@
},
"@libp2p/bootstrap": {
"version": "5.0.0",
"dev": true,
"requires": {
"@libp2p/interface-peer-discovery": "^1.0.1",
"@libp2p/interface-peer-info": "^1.0.3",
@ -28114,6 +28110,26 @@
"@multiformats/multiaddr": "^11.0.0"
}
},
"@libp2p/components": {
"version": "3.1.1",
"requires": {
"@libp2p/interface-address-manager": "^2.0.0",
"@libp2p/interface-connection": "^3.0.1",
"@libp2p/interface-connection-manager": "^1.1.0",
"@libp2p/interface-content-routing": "^1.0.2",
"@libp2p/interface-dht": "^1.0.1",
"@libp2p/interface-metrics": "^3.0.0",
"@libp2p/interface-peer-id": "^1.0.4",
"@libp2p/interface-peer-routing": "^1.0.1",
"@libp2p/interface-peer-store": "^1.2.2",
"@libp2p/interface-pubsub": "^3.0.0",
"@libp2p/interface-registrar": "^2.0.3",
"@libp2p/interface-transport": "^2.0.0",
"@libp2p/interfaces": "^3.0.3",
"err-code": "^3.0.1",
"interface-datastore": "^7.0.0"
}
},
"@libp2p/connection": {
"version": "4.0.2",
"requires": {
@ -29885,7 +29901,6 @@
"version": "file:packages/create",
"requires": {
"@chainsafe/libp2p-noise": "^10.1.0",
"@libp2p/bootstrap": "^5.0.0",
"@libp2p/interface-address-manager": "^2.0.1",
"@libp2p/interface-connection": "^3.0.3",
"@libp2p/interface-connection-manager": "^1.3.1",
@ -29907,6 +29922,7 @@
"@typescript-eslint/eslint-plugin": "^5.8.1",
"@typescript-eslint/parser": "^5.8.1",
"@waku/core": "*",
"@waku/dns-discovery": "*",
"@waku/interfaces": "*",
"@waku/peer-exchange": "*",
"cspell": "^6.17.0",
@ -30144,6 +30160,7 @@
"version": "file:packages/tests",
"requires": {
"@libp2p/bootstrap": "^5.0.0",
"@libp2p/components": "^3.1.1",
"@typescript-eslint/eslint-plugin": "^5.8.1",
"@typescript-eslint/parser": "^5.8.1",
"@waku/byte-utils": "*",
@ -35646,7 +35663,7 @@
"dev": true
},
"json5": {
"version": "1.0.1",
"version": "1.0.2",
"dev": true,
"requires": {
"minimist": "^1.2.0"
@ -40597,11 +40614,6 @@
"esbuild-windows-arm64": "0.14.39"
}
},
"esbuild-darwin-64": {
"version": "0.14.39",
"dev": true,
"optional": true
},
"globby": {
"version": "13.1.2",
"dev": true,
@ -43186,7 +43198,7 @@
"dev": true
},
"ua-parser-js": {
"version": "0.7.32",
"version": "0.7.33",
"dev": true
},
"uc.micro": {

View File

@ -51,7 +51,6 @@
},
"dependencies": {
"@chainsafe/libp2p-noise": "^10.1.0",
"@libp2p/bootstrap": "^5.0.0",
"@libp2p/interface-address-manager": "^2.0.1",
"@libp2p/interface-connection": "^3.0.3",
"@libp2p/interface-connection-manager": "^1.3.1",
@ -68,6 +67,7 @@
"@libp2p/mplex": "^7.0.0",
"@libp2p/websockets": "^5.0.0",
"@waku/core": "*",
"@waku/dns-discovery": "*",
"@waku/peer-exchange": "*",
"interface-datastore": "^7.0.1"
},

View File

@ -1,5 +1,4 @@
import { noise } from "@chainsafe/libp2p-noise";
import { bootstrap } from "@libp2p/bootstrap";
import type { PeerDiscovery } from "@libp2p/interface-peer-discovery";
import { mplex } from "@libp2p/mplex";
import { webSockets } from "@libp2p/websockets";
@ -14,7 +13,7 @@ import {
wakuStore,
} from "@waku/core";
import { DefaultUserAgent } from "@waku/core";
import { getPredefinedBootstrapNodes } from "@waku/core/lib/predefined_bootstrap_nodes";
import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
import type { FullNode, IRelay, LightNode, RelayNode } from "@waku/interfaces";
import { wakuPeerExchange } from "@waku/peer-exchange";
import type { Libp2p } from "libp2p";
@ -22,6 +21,12 @@ import { createLibp2p, Libp2pOptions } from "libp2p";
import type { Libp2pComponents } from "./libp2p_components.js";
const DEFAULT_NODE_REQUIREMENTS = {
lightPush: 1,
filter: 1,
store: 1,
};
export { Libp2pComponents };
export interface CreateOptions {
@ -165,7 +170,7 @@ export async function createFullNode(
export function defaultPeerDiscovery(): (
components: Libp2pComponents
) => PeerDiscovery {
return bootstrap({ list: getPredefinedBootstrapNodes() });
return wakuDnsDiscovery(enrTree["PROD"], DEFAULT_NODE_REQUIREMENTS);
}
export async function defaultLibp2p(

View File

@ -3,6 +3,8 @@ import { expect } from "chai";
import { DnsClient, DnsNodeDiscovery } from "./dns.js";
import testData from "./testdata.json" assert { type: "json" };
import { enrTree } from "./index.js";
const mockData = testData.dns;
const host = "nodes.example.org";
@ -260,9 +262,6 @@ describe("DNS Node Discovery w/ capabilities", () => {
});
describe("DNS Node Discovery [live data]", function () {
const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM";
const fqdn = "test.waku.nodes.status.im";
const enrTree = `enrtree://${publicKey}@${fqdn}`;
const maxQuantity = 3;
before(function () {
@ -275,7 +274,30 @@ describe("DNS Node Discovery [live data]", function () {
this.timeout(10000);
// Google's dns server address. Needs to be set explicitly to run in CI
const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
const peers = await dnsNodeDiscovery.getPeers([enrTree], {
const peers = await dnsNodeDiscovery.getPeers([enrTree.TEST], {
relay: maxQuantity,
store: maxQuantity,
filter: maxQuantity,
lightPush: maxQuantity,
});
expect(peers.length).to.eq(maxQuantity);
const multiaddrs = peers.map((peer) => peer.multiaddrs).flat();
const seen: string[] = [];
for (const ma of multiaddrs) {
expect(ma).to.not.be.undefined;
expect(seen).to.not.include(ma!.toString());
seen.push(ma!.toString());
}
});
it(`should retrieve ${maxQuantity} multiaddrs for prod.waku.nodes.status.im`, async function () {
this.timeout(10000);
// Google's dns server address. Needs to be set explicitly to run in CI
const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
const peers = await dnsNodeDiscovery.getPeers([enrTree.PROD], {
relay: maxQuantity,
store: maxQuantity,
filter: maxQuantity,

View File

@ -5,7 +5,7 @@ import type {
import { symbol } from "@libp2p/interface-peer-discovery";
import type { PeerInfo } from "@libp2p/interface-peer-info";
import { CustomEvent, EventEmitter } from "@libp2p/interfaces/events";
import type { IEnr } from "@waku/interfaces";
import type { IEnr, PeerExchangeComponents } from "@waku/interfaces";
import { multiaddrsToPeerInfo } from "@waku/libp2p-utils";
import debug from "debug";
@ -15,6 +15,40 @@ export { NodeCapabilityCount };
const log = debug("waku:peer-discovery-dns");
const enrTree = {
TEST: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@test.waku.nodes.status.im",
PROD: "enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@prod.waku.nodes.status.im",
};
const DEFAULT_BOOTSTRAP_TAG_NAME = "bootstrap";
const DEFAULT_BOOTSTRAP_TAG_VALUE = 50;
const DEFAULT_BOOTSTRAP_TAG_TTL = 120000;
export interface Options {
/**
* ENR URL to use for DNS discovery
*/
enrUrl: string;
/**
* Specifies what type of nodes are wanted from the discovery process
*/
wantedNodeCapabilityCount: Partial<NodeCapabilityCount>;
/**
* Tag a bootstrap peer with this name before "discovering" it (default: 'bootstrap')
*/
tagName?: string;
/**
* The bootstrap peer tag will have this value (default: 50)
*/
tagValue?: number;
/**
* Cause the bootstrap peer tag to be removed after this number of ms (default: 2 minutes)
*/
tagTTL?: number;
}
/**
* Parse options and expose function to return bootstrap peer addresses.
*/
@ -24,19 +58,17 @@ export class PeerDiscoveryDns
{
private readonly nextPeer: () => AsyncGenerator<IEnr>;
private _started: boolean;
private _components: PeerExchangeComponents;
private _options: Options;
/**
* @param enrUrl An EIP-1459 ENR Tree URL. For example:
* "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev"
* @param wantedNodeCapabilityCount Specifies what node capabilities
* (protocol) must be returned.
*/
constructor(
enrUrl: string,
wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
) {
constructor(components: PeerExchangeComponents, options: Options) {
super();
this._started = false;
this._components = components;
this._options = options;
const { enrUrl, wantedNodeCapabilityCount } = options;
log("Use following EIP-1459 ENR Tree URL: ", enrUrl);
const dns = DnsNodeDiscovery.dnsOverHttp();
@ -58,7 +90,15 @@ export class PeerDiscoveryDns
for await (const peer of this.nextPeer()) {
if (!this._started) return;
const peerInfos = multiaddrsToPeerInfo(peer.getFullMultiaddrs());
peerInfos.forEach((peerInfo) => {
peerInfos.forEach(async (peerInfo) => {
await this._components.peerStore.tagPeer(
peerInfo.id,
DEFAULT_BOOTSTRAP_TAG_NAME,
{
value: this._options.tagValue ?? DEFAULT_BOOTSTRAP_TAG_VALUE,
ttl: this._options.tagTTL ?? DEFAULT_BOOTSTRAP_TAG_TTL,
}
);
this.dispatchEvent(
new CustomEvent<PeerInfo>("peer", { detail: peerInfo })
);
@ -85,8 +125,11 @@ export class PeerDiscoveryDns
export function wakuDnsDiscovery(
enrUrl: string,
wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
): () => PeerDiscoveryDns {
return () => new PeerDiscoveryDns(enrUrl, wantedNodeCapabilityCount);
): (components: PeerExchangeComponents) => PeerDiscoveryDns {
return (components: PeerExchangeComponents) =>
new PeerDiscoveryDns(components, { enrUrl, wantedNodeCapabilityCount });
}
export { DnsNodeDiscovery, SearchContext, DnsClient } from "./dns.js";
export { enrTree };

View File

@ -58,8 +58,8 @@
"@waku/core": "*",
"@waku/enr": "*",
"@waku/interfaces": "*",
"@waku/proto": "*",
"@waku/libp2p-utils": "*",
"@waku/proto": "*",
"debug": "^4.3.4",
"it-all": "^1.0.6",
"it-length-prefixed": "^8.0.2",

View File

@ -55,18 +55,22 @@
"node": ">=16"
},
"dependencies": {
"app-root-path": "^3.0.0",
"portfinder": "^1.0.28",
"p-timeout": "^6.0.0",
"tail": "^2.2.0",
"@libp2p/components": "^3.1.1",
"@waku/byte-utils": "*",
"@waku/core": "*",
"@waku/create": "*",
"@waku/dns-discovery": "*",
"@waku/enr": "*",
"@waku/interfaces": "*",
"@waku/message-encryption": "*",
"@waku/peer-exchange": "*",
"@waku/dns-discovery": "*"
"app-root-path": "^3.0.0",
"chai": "^4.3.6",
"debug": "^4.3.4",
"mocha": "^9.1.3",
"p-timeout": "^6.0.0",
"portfinder": "^1.0.28",
"tail": "^2.2.0"
},
"devDependencies": {
"mocha": "^9.1.3",

View File

@ -1,20 +1,38 @@
import { Components } from "@libp2p/components";
import tests from "@libp2p/interface-peer-discovery-compliance-tests";
import { Peer } from "@libp2p/interface-peer-store";
import { createSecp256k1PeerId } from "@libp2p/peer-id-factory";
import { PersistentPeerStore } from "@libp2p/peer-store";
import { createLightNode } from "@waku/create";
import { DnsNodeDiscovery, wakuDnsDiscovery } from "@waku/dns-discovery";
import {
DnsNodeDiscovery,
enrTree,
PeerDiscoveryDns,
wakuDnsDiscovery,
} from "@waku/dns-discovery";
import { expect } from "chai";
import { MemoryDatastore } from "datastore-core";
const publicKey = "AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM";
const fqdn = "test.waku.nodes.status.im";
const enrTree = `enrtree://${publicKey}@${fqdn}`;
const maxQuantity = 3;
describe("DNS Discovery: Compliance Test", async function () {
this.timeout(10000);
tests({
async setup() {
return wakuDnsDiscovery(enrTree, {
filter: 1,
})();
// create libp2p mock peerStore
const components = new Components({
peerStore: new PersistentPeerStore({
peerId: await createSecp256k1PeerId(),
datastore: new MemoryDatastore(),
}),
});
return new PeerDiscoveryDns(components, {
enrUrl: enrTree["PROD"],
wantedNodeCapabilityCount: {
filter: 1,
},
});
},
async teardown() {
//
@ -42,22 +60,37 @@ describe("DNS Node Discovery [live data]", function () {
const waku = await createLightNode({
libp2p: {
peerDiscovery: [wakuDnsDiscovery(enrTree, nodeRequirements)],
peerDiscovery: [wakuDnsDiscovery(enrTree["PROD"], nodeRequirements)],
},
});
await waku.start();
const peersFound = await waku.libp2p.peerStore.all();
expect(peersFound.length).to.eq(maxQuantity);
const allPeers = await waku.libp2p.peerStore.all();
const dnsPeers: Peer[] = [];
for (const peer of allPeers) {
const tags = await waku.libp2p.peerStore.getTags(peer.id);
let hasTag = false;
for (const tag of tags) {
hasTag = tag.name === "bootstrap";
if (hasTag) {
dnsPeers.push(peer);
break;
}
}
expect(hasTag).to.be.eq(true);
}
expect(dnsPeers.length).to.eq(maxQuantity);
});
it(`should retrieve ${maxQuantity} multiaddrs for prod.nodes.status.im`, async function () {
it(`should retrieve ${maxQuantity} multiaddrs for test.waku.nodes.status.im`, async function () {
this.timeout(10000);
// Google's dns server address. Needs to be set explicitly to run in CI
const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp();
const peers = await dnsNodeDiscovery.getPeers([enrTree], {
const peers = await dnsNodeDiscovery.getPeers([enrTree["PROD"]], {
relay: maxQuantity,
store: maxQuantity,
filter: maxQuantity,