From f4f6b4a8086dd96b0d8d79475a714be385038bcf Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Fri, 24 Sep 2021 15:43:42 +1000 Subject: [PATCH 01/16] Create discovery submodule --- src/lib/{discovery.spec.ts => discovery/index.spec.ts} | 2 +- src/lib/{discovery.ts => discovery/index.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/lib/{discovery.spec.ts => discovery/index.spec.ts} (94%) rename src/lib/{discovery.ts => discovery/index.ts} (100%) diff --git a/src/lib/discovery.spec.ts b/src/lib/discovery/index.spec.ts similarity index 94% rename from src/lib/discovery.spec.ts rename to src/lib/discovery/index.spec.ts index ff4b7a7fa2..0969766e95 100644 --- a/src/lib/discovery.spec.ts +++ b/src/lib/discovery/index.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; -import { getPseudoRandomSubset } from './discovery'; +import { getPseudoRandomSubset } from './index'; describe('Discovery', () => { it('returns all values when wanted number matches available values', function () { diff --git a/src/lib/discovery.ts b/src/lib/discovery/index.ts similarity index 100% rename from src/lib/discovery.ts rename to src/lib/discovery/index.ts From e244bae03de9ed3f6a881a0b71f5cbf18bf57b61 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 15:26:52 +1100 Subject: [PATCH 02/16] Renamed `getBootstrapNodes` to `getNodesFromHostedJson` So that the name better matches the function's behaviour. --- CHANGELOG.md | 1 + src/index.ts | 2 +- src/lib/discovery/hosted_json.ts | 74 ++++++++++++++++++++++++++++++++ src/lib/discovery/index.ts | 69 +---------------------------- src/lib/waku.ts | 6 +-- 5 files changed, 80 insertions(+), 72 deletions(-) create mode 100644 src/lib/discovery/hosted_json.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index af97df1838..8901263351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Test: Upgrade nim-waku node to v0.6. +- **Breaking**: Renamed `getBootstrapNodes` to `getNodesFromHostedJson`. - Minimum node version changed to 16. ### Fixed diff --git a/src/index.ts b/src/index.ts index 435ec291ae..4887685ba3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -export { getBootstrapNodes } from './lib/discovery'; +export { getNodesFromHostedJson, getPseudoRandomSubset } from './lib/discovery'; export * as utils from './lib/utils'; diff --git a/src/lib/discovery/hosted_json.ts b/src/lib/discovery/hosted_json.ts new file mode 100644 index 0000000000..bfabea3aca --- /dev/null +++ b/src/lib/discovery/hosted_json.ts @@ -0,0 +1,74 @@ +/** + * GET list of nodes from remote HTTP host. + * + * Default behavior is to return nodes hosted by Status. + * + * @param path The property path to access the node list. The result should be + * a string, a string array or an object. If the result is an object then the + * values of the objects are used as multiaddresses. For example, if the GET + * request returns `{ foo: { bar: [address1, address2] } }` then `path` should be + * `[ "foo", "bar" ]`. + * @param url Remote host containing bootstrap peers in JSON format. + * @param wantedNumber The number of connections desired. Defaults to [DefaultWantedNumber]. + * + * @returns An array of multiaddresses. + * @throws If the remote host is unreachable or the response cannot be parsed + * according to the passed _path_. + */ +import axios from 'axios'; +import debug from 'debug'; +import { Multiaddr } from 'multiaddr'; + +import { getPseudoRandomSubset } from './index'; +const dbg = debug('waku:discovery'); + +const DefaultWantedNumber = 1; + +export async function getNodesFromHostedJson( + path: string[] = ['fleets', 'wakuv2.prod', 'waku-websocket'], + url = 'https://fleets.status.im/', + wantedNumber: number = DefaultWantedNumber +): Promise { + if (wantedNumber <= 0) { + return []; + } + + const res = await axios.get(url, { + headers: { 'Content-Type': 'application/json' }, + }); + + let nodes = res.data; + + for (const prop of path) { + if (nodes[prop] === undefined) { + dbg( + `Failed to retrieve bootstrap nodes: ${prop} does not exist on `, + nodes + ); + throw `Failed to retrieve bootstrap nodes: ${prop} does not exist on ${JSON.stringify( + nodes + )}`; + } + nodes = nodes[prop]; + } + + if (Array.isArray(nodes)) { + return getPseudoRandomSubset(nodes, wantedNumber).map( + (node: string) => node + ); + } + + if (typeof nodes === 'string') { + return [nodes]; + } + + if (typeof nodes === 'object') { + nodes = Object.values(nodes) as string[]; + nodes = nodes.map((node: string) => new Multiaddr(node)); + return getPseudoRandomSubset(nodes, wantedNumber); + } + + throw `Failed to retrieve bootstrap nodes: response format is not supported: ${JSON.stringify( + nodes + )}`; +} diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index 6fabcf5fa6..09eeb32682 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -1,73 +1,6 @@ -import axios from 'axios'; -import debug from 'debug'; import { shuffle } from 'libp2p-gossipsub/src/utils'; -const dbg = debug('waku:discovery'); - -const DefaultWantedNumber = 1; - -/** - * GET list of nodes from remote HTTP host. - * - * Default behavior is to return nodes hosted by Status. - * - * @param path The property path to access the node list. The result should be - * a string, a string array or an object. If the result is an object then the - * values of the objects are used as multiaddresses. For example, if the GET - * request returns `{ foo: { bar: [address1, address2] } }` then `path` should be - * `[ "foo", "bar" ]`. - * @param url Remote host containing bootstrap peers in JSON format. - * @param wantedNumber The number of connections desired. Defaults to [DefaultWantedNumber]. - * - * @returns An array of multiaddresses. - * @throws If the remote host is unreachable or the response cannot be parsed - * according to the passed _path_. - */ -export async function getBootstrapNodes( - path: string[] = ['fleets', 'wakuv2.prod', 'waku-websocket'], - url = 'https://fleets.status.im/', - wantedNumber: number = DefaultWantedNumber -): Promise { - if (wantedNumber <= 0) { - return []; - } - - const res = await axios.get(url, { - headers: { 'Content-Type': 'application/json' }, - }); - - let nodes = res.data; - - for (const prop of path) { - if (nodes[prop] === undefined) { - dbg( - `Failed to retrieve bootstrap nodes: ${prop} does not exist on `, - nodes - ); - throw `Failed to retrieve bootstrap nodes: ${prop} does not exist on ${JSON.stringify( - nodes - )}`; - } - nodes = nodes[prop]; - } - - if (Array.isArray(nodes)) { - return getPseudoRandomSubset(nodes, wantedNumber); - } - - if (typeof nodes === 'string') { - return [nodes]; - } - - if (typeof nodes === 'object') { - nodes = Object.values(nodes); - return getPseudoRandomSubset(nodes, wantedNumber); - } - - throw `Failed to retrieve bootstrap nodes: response format is not supported: ${JSON.stringify( - nodes - )}`; -} +export { getNodesFromHostedJson } from './hosted_json'; export function getPseudoRandomSubset( values: string[], diff --git a/src/lib/waku.ts b/src/lib/waku.ts index 6900ab6e07..f5da1c5d3e 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -18,7 +18,7 @@ import Ping from 'libp2p/src/ping'; import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; -import { getBootstrapNodes } from './discovery'; +import { getNodesFromHostedJson } from './discovery'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; import { WakuMessage } from './waku_message'; @@ -87,7 +87,7 @@ export interface CreateOptions { * Use libp2p-bootstrap to discover and connect to new nodes. * * You can pass: - * - `true` to use {@link getBootstrapNodes}, + * - `true` to use {@link getNodesFromHostedJson}, * - an array of multiaddresses, * - a function that returns an array of multiaddresses (or Promise of). * @@ -192,7 +192,7 @@ export class Waku { let bootstrap: undefined | (() => string[] | Promise); if (options.bootstrap === true) { - bootstrap = getBootstrapNodes; + bootstrap = getNodesFromHostedJson; } else if (Array.isArray(options.bootstrap)) { bootstrap = (): string[] => { return options.bootstrap as string[]; From e47335f4c0cfad203a7bd00eb1c85a053696595a Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 11:33:26 +1100 Subject: [PATCH 03/16] Implement DNS Discovery and ENR tree --- .cspell.json | 8 +- package-lock.json | 101 ++++++++++++++ package.json | 2 + src/lib/discovery/bootstrap.ts | 115 ++++++++++++++++ src/lib/discovery/dns.spec.ts | 197 +++++++++++++++++++++++++++ src/lib/discovery/dns.ts | 201 ++++++++++++++++++++++++++++ src/lib/discovery/dns_over_https.ts | 60 +++++++++ src/lib/discovery/enrtree.spec.ts | 89 ++++++++++++ src/lib/discovery/enrtree.ts | 123 +++++++++++++++++ src/lib/discovery/index.ts | 1 + src/lib/discovery/testdata.json | 18 +++ src/lib/enr/enr.spec.ts | 4 +- src/lib/enr/enr.ts | 12 +- src/lib/utils.ts | 6 + src/lib/waku.ts | 19 +-- 15 files changed, 932 insertions(+), 24 deletions(-) create mode 100644 src/lib/discovery/bootstrap.ts create mode 100644 src/lib/discovery/dns.spec.ts create mode 100644 src/lib/discovery/dns.ts create mode 100644 src/lib/discovery/dns_over_https.ts create mode 100644 src/lib/discovery/enrtree.spec.ts create mode 100644 src/lib/discovery/enrtree.ts create mode 100644 src/lib/discovery/testdata.json diff --git a/.cspell.json b/.cspell.json index ab5c739a25..3a3dd3ab16 100644 --- a/.cspell.json +++ b/.cspell.json @@ -23,8 +23,9 @@ "Dscore", "ecies", "editorconfig", - "ENR", - "ENRs", + "enr", + "enrs", + "enrtree", "ephem", "esnext", "ethersproject", @@ -61,6 +62,7 @@ "muxer", "mvps", "nodekey", + "opendns", "peerhave", "prettierignore", "proto", @@ -71,9 +73,11 @@ "rlnrelay", "roadmap", "sandboxed", + "scanf", "secio", "seckey", "secp", + "sscanf", "staticnode", "statusim", "submodule", diff --git a/package-lock.json b/package-lock.json index 8d266c697a..aa3d8afbe0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", @@ -2106,6 +2108,11 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "node_modules/@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -3118,6 +3125,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "node_modules/@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -6183,6 +6198,42 @@ "receptacle": "^1.3.2" } }, + "node_modules/dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + }, + "bin": { + "dns-query": "bin/dns-query" + } + }, + "node_modules/dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "dependencies": { + "dns-packet": "^5.2.4" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -8368,6 +8419,11 @@ "node": ">= 8" } }, + "node_modules/hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -19349,6 +19405,11 @@ "chalk": "^4.0.0" } }, + "@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -20233,6 +20294,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "requires": { + "@types/node": "*" + } + }, "@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -22697,6 +22766,33 @@ "receptacle": "^1.3.2" } }, + "dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.1" + } + }, + "dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + } + }, + "dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "requires": { + "dns-packet": "^5.2.4" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -24357,6 +24453,11 @@ } } }, + "hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", diff --git a/package.json b/package.json index 7202a433f4..5686982d5d 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts new file mode 100644 index 0000000000..6ba0860bc0 --- /dev/null +++ b/src/lib/discovery/bootstrap.ts @@ -0,0 +1,115 @@ +import debug from 'debug'; + +import { DnsNodeDiscovery } from './dns'; + +import { getNodesFromHostedJson, getPseudoRandomSubset } from './index'; + +const dbg = debug('waku:discovery:bootstrap'); + +const DefaultMaxPeers = 1; + +/** + * Setup discovery method used to bootstrap. + * + * Only one method is used. `default`, `peers`, `getPeers` and `enr` options are mutually exclusive. + */ +export interface BootstrapOptions { + /** + * The maximum of peers to connect to as part of the bootstrap process. + * + * @default 1 + */ + maxPeers?: number; + /** + * Use the default discovery method. + * + * The default discovery method is likely to change overtime as new discovery + * methods are implemented. + * + * @default false + */ + default?: boolean; + /** + * Multiaddrs of peers to connect to. + */ + peers?: string[]; + /** + * Getter that retrieve multiaddrs of peers to connect to. + */ + getPeers?: () => Promise; + /** + * An EIP-1459 ENR Tree URL. For example: + * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev" + */ + enrUrl?: string; +} + +/** + * Parse the bootstrap options and returns an async function that returns node addresses upon invocation. + */ +export function parseBootstrap( + options: + | BootstrapOptions + | boolean + | string[] + | (() => string[] | Promise) +): undefined | (() => Promise) { + if ( + Object.keys(options).includes('default') || + Object.keys(options).includes('maxPeers') || + Object.keys(options).includes('peers') || + Object.keys(options).includes('getPeers') || + Object.keys(options).includes('enrUrl') + ) { + const opts = options as unknown as BootstrapOptions; + const maxPeers = opts.maxPeers || DefaultMaxPeers; + + if (opts.default) { + dbg('Bootstrap: Use hosted list of peers.'); + + return getNodesFromHostedJson.bind({}, undefined, undefined, maxPeers); + } else if (opts.peers !== undefined && opts.peers.length > 0) { + dbg('Bootstrap: Use provided list of peers.'); + + const allPeers: string[] = opts.peers; + return (): Promise => { + const peers = getPseudoRandomSubset(allPeers, maxPeers); + return Promise.resolve(peers); + }; + } else if (typeof opts.getPeers === 'function') { + dbg('Bootstrap: Use provided getPeers function.'); + const getPeers = opts.getPeers; + + return async (): Promise => { + const allPeers = await getPeers(); + return getPseudoRandomSubset(allPeers, maxPeers); + }; + } else if (opts.enrUrl) { + const enrUrl = opts.enrUrl; + dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); + + const dns = DnsNodeDiscovery.dnsOverHttp(); + // TODO: The `toString` is incorrect. + return (): Promise => + dns + .getPeers(maxPeers, [enrUrl]) + .then((peers) => peers.map((peer) => peer.toString())); + } + } else { + dbg( + 'WARN: This bootstrap method will be deprecated, use `BootstrapOptions` instead' + ); + if (options === true) { + return getNodesFromHostedJson; + } else if (Array.isArray(options)) { + return (): Promise => { + return Promise.resolve(options as string[]); + }; + } else if (typeof options === 'function') { + return async (): Promise => { + return options(); + }; + } + } + return; +} diff --git a/src/lib/discovery/dns.spec.ts b/src/lib/discovery/dns.spec.ts new file mode 100644 index 0000000000..2563c70ade --- /dev/null +++ b/src/lib/discovery/dns.spec.ts @@ -0,0 +1,197 @@ +import { expect } from 'chai'; + +import { DnsClient, DnsNodeDiscovery } from './dns'; +import testData from './testdata.json'; + +const mockData = testData.dns; + +const host = 'nodes.example.org'; +const rootDomain = 'JORXBYVVM7AEKETX5DGXW44EAY'; +const branchDomainA = 'D2SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainB = 'D3SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainC = 'D4SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainD = 'D5SNLTAGWNQ34NTQTPHNZDECFU'; +const partialBranchA = 'AAAA'; +const partialBranchB = 'BBBB'; +const singleBranch = `enrtree-branch:${branchDomainA}`; +const doubleBranch = `enrtree-branch:${branchDomainA},${branchDomainB}`; +const multiComponentBranch = [ + `enrtree-branch:${branchDomainA},${partialBranchA}`, + `${partialBranchB},${branchDomainB}`, +]; + +// Note: once td.when is asked to throw for an input it will always throw. +// Input can't be re-used for a passing case. +const errorBranchA = `enrtree-branch:${branchDomainC}`; +const errorBranchB = `enrtree-branch:${branchDomainD}`; + +/** + * Mocks DNS resolution. + */ +class MockDNS implements DnsClient { + fqdnRes: Map; + fqdnThrows: string[]; + + constructor() { + this.fqdnRes = new Map(); + this.fqdnThrows = []; + } + + addRes(fqdn: string, res: string[]): void { + this.fqdnRes.set(fqdn, res); + } + + addThrow(fqdn: string): void { + this.fqdnThrows.push(fqdn); + } + + resolveTXT(fqdn: string): Promise { + if (this.fqdnThrows.includes(fqdn)) throw 'Mock DNS throws.'; + + const res = this.fqdnRes.get(fqdn); + + if (!res) throw `Mock DNS could not resolve ${fqdn}`; + + return Promise.resolve(res); + } +} + +describe('DNS Node Discovery', () => { + let mockDns: MockDNS; + + beforeEach(() => { + mockDns = new MockDNS(); + mockDns.addRes(host, [mockData.enrRoot]); + }); + + it('retrieves a single peer', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(1); + expect(peers[0].ip).to.eq('45.77.40.127'); + expect(peers[0].tcp).to.eq(30303); + }); + + it('retrieves all peers (2) when maxQuantity larger than DNS tree size', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [doubleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrB]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(2); + expect(peers[0].ip).to.not.eq(peers[1].ip); + }); + + it('retrieves all peers (3) when branch entries are composed of multiple strings', async function () { + mockDns.addRes(`${rootDomain}.${host}`, multiComponentBranch); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enr]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrA]); + mockDns.addRes(`${partialBranchA}${partialBranchB}.${host}`, [ + mockData.enrB, + ]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(3); + expect(peers[0].ip).to.not.eq(peers[1].ip); + expect(peers[0].ip).to.not.eq(peers[2].ip); + expect(peers[1].ip).to.not.eq(peers[2].ip); + }); + + it('it tolerates circular branch references', async function () { + // root --> branchA + // branchA --> branchA + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [singleBranch]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + }); + + it('recovers when dns.resolve returns empty', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + + // Empty response case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + let peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + + // No TXT records case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores domain fetching errors', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchA]); + mockDns.addThrow(`${branchDomainC}.${host}`); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores unrecognized TXT record formats', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [mockData.enrBranchBadPrefix]); + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('caches peers it previously fetched', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchB]); + mockDns.addRes(`${branchDomainD}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peersA = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersA.length).to.eq(1); + + // Specify that a subsequent network call retrieving the same peer should throw. + // This test passes only if the peer is fetched from cache + mockDns.addThrow(`${branchDomainD}.${host}`); + + const peersB = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersB.length).to.eq(1); + expect(peersA[0].ip).to.eq(peersB[0].ip); + }); +}); + +describe('DNS Node Discovery [live data]', function () { + const publicKey = 'AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C'; + const fqdn = 'test.nodes.vac.dev'; + const enrTree = `enrtree://${publicKey}@${fqdn}`; + const ipTestRegex = /^\d+\.\d+\.\d+\.\d+$/; + const maxQuantity = 3; + + it(`should retrieve ${maxQuantity} PeerInfos for test.nodes.vac.dev`, async function () { + this.timeout(5000); + // Google's dns server address. Needs to be set explicitly to run in CI + const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); + const peers = await dnsNodeDiscovery.getPeers(maxQuantity, [enrTree]); + + expect(peers.length).to.eq(maxQuantity); + + // TODO: Test multiaddrs entry + console.log(peers.map((peer) => peer.multiaddrs)); + + const seen: string[] = []; + for (const peer of peers) { + expect(peer!.ip!).to.match(ipTestRegex); + expect(seen).to.not.include(peer!.ip!); + seen.push(peer!.ip!); + } + }); +}); diff --git a/src/lib/discovery/dns.ts b/src/lib/discovery/dns.ts new file mode 100644 index 0000000000..ba3efd00f6 --- /dev/null +++ b/src/lib/discovery/dns.ts @@ -0,0 +1,201 @@ +import assert from 'assert'; + +import { debug } from 'debug'; + +import { ENR } from '../enr'; + +import { DnsOverHttps, Endpoints } from './dns_over_https'; +import { ENRTree } from './enrtree'; + +const dbg = debug('waku:discovery:dns'); + +type SearchContext = { + domain: string; + publicKey: string; + visits: { [key: string]: boolean }; +}; + +export interface DnsClient { + resolveTXT: (domain: string) => Promise; +} + +export class DnsNodeDiscovery { + private readonly dns: DnsClient; + private readonly _DNSTreeCache: { [key: string]: string }; + private readonly _errorTolerance: number = 10; + + public static dnsOverHttp(endpoints?: Endpoints): DnsNodeDiscovery { + const dnsClient = new DnsOverHttps(endpoints); + return new DnsNodeDiscovery(dnsClient); + } + + /** + * Returns a list of verified peers listed in an EIP-1459 DNS tree. Method may + * return fewer peers than requested if `maxQuantity` is larger than the number + * of ENR records or the number of errors/duplicate peers encountered by randomized + * search exceeds `maxQuantity` plus the `errorTolerance` factor. + */ + async getPeers(maxQuantity: number, enrTreeUrls: string[]): Promise { + let totalSearches = 0; + const peers: ENR[] = []; + + const networkIndex = Math.floor(Math.random() * enrTreeUrls.length); + const { publicKey, domain } = ENRTree.parseTree(enrTreeUrls[networkIndex]); + + while ( + peers.length < maxQuantity && + totalSearches < maxQuantity + this._errorTolerance + ) { + const context: SearchContext = { + domain, + publicKey, + visits: {}, + }; + + const peer = await this._search(domain, context); + + if (peer && isNewPeer(peer, peers)) { + peers.push(peer); + dbg( + `got new peer candidate from DNS address=${peer.nodeId}@${peer.ip}` + ); + } + + totalSearches++; + } + return peers; + } + + public constructor(dns: DnsClient) { + this._DNSTreeCache = {}; + this.dns = dns; + } + + /** + * Runs a recursive, randomized descent of the DNS tree to retrieve a single + * ENR record as an ENR. Returns null if parsing or DNS resolution fails. + */ + private async _search( + subdomain: string, + context: SearchContext + ): Promise { + const entry = await this._getTXTRecord(subdomain, context); + context.visits[subdomain] = true; + + let next: string; + let branches: string[]; + + const entryType = getEntryType(entry); + try { + switch (entryType) { + case ENRTree.ROOT_PREFIX: + next = ENRTree.parseAndVerifyRoot(entry, context.publicKey); + return await this._search(next, context); + case ENRTree.BRANCH_PREFIX: + branches = ENRTree.parseBranch(entry); + next = selectRandomPath(branches, context); + return await this._search(next, context); + case ENRTree.RECORD_PREFIX: + return ENR.decodeTxt(entry); + default: + return null; + } + } catch (error) { + dbg( + `Failed to search DNS tree ${entryType} at subdomain ${subdomain}: ${error}` + ); + return null; + } + } + + /** + * Retrieves the TXT record stored at a location from either + * this DNS tree cache or via DNS query + */ + private async _getTXTRecord( + subdomain: string, + context: SearchContext + ): Promise { + if (this._DNSTreeCache[subdomain]) { + return this._DNSTreeCache[subdomain]; + } + + // Location is either the top level tree entry host or a subdomain of it. + const location = + subdomain !== context.domain + ? `${subdomain}.${context.domain}` + : context.domain; + + const response = await this.dns.resolveTXT(location); + + assert( + response.length, + 'Received empty result array while fetching TXT record' + ); + assert(response[0].length, 'Received empty TXT record'); + + // Branch entries can be an array of strings of comma delimited subdomains, with + // some subdomain strings split across the array elements + const result = response.join(''); + + this._DNSTreeCache[subdomain] = result; + return result; + } +} + +function getEntryType(entry: string): string { + if (entry.startsWith(ENRTree.ROOT_PREFIX)) return ENRTree.ROOT_PREFIX; + if (entry.startsWith(ENRTree.BRANCH_PREFIX)) return ENRTree.BRANCH_PREFIX; + if (entry.startsWith(ENRTree.RECORD_PREFIX)) return ENRTree.RECORD_PREFIX; + + return ''; +} + +/** + * Returns a randomly selected subdomain string from the list provided by a branch + * entry record. + * + * The client must track subdomains which are already resolved to avoid + * going into an infinite loop b/c branch entries can contain + * circular references. It’s in the client’s best interest to traverse the + * tree in random order. + */ +function selectRandomPath(branches: string[], context: SearchContext): string { + // Identify domains already visited in this traversal of the DNS tree. + // Then filter against them to prevent cycles. + const circularRefs: { [key: number]: boolean } = {}; + for (const [idx, subdomain] of branches.entries()) { + if (context.visits[subdomain]) { + circularRefs[idx] = true; + } + } + // If all possible paths are circular... + if (Object.keys(circularRefs).length === branches.length) { + throw new Error('Unresolvable circular path detected'); + } + + // Randomly select a viable path + let index; + do { + index = Math.floor(Math.random() * branches.length); + } while (circularRefs[index]); + + return branches[index]; +} + +/** + * @returns false if candidate peer already exists in the + * current collection of peers based on the node id value; + * true otherwise. + */ +function isNewPeer(peer: ENR | null, peers: ENR[]): boolean { + if (!peer || !peer.nodeId) return false; + + for (const existingPeer of peers) { + if (peer.nodeId === existingPeer.nodeId) { + return false; + } + } + + return true; +} diff --git a/src/lib/discovery/dns_over_https.ts b/src/lib/discovery/dns_over_https.ts new file mode 100644 index 0000000000..d71fb80130 --- /dev/null +++ b/src/lib/discovery/dns_over_https.ts @@ -0,0 +1,60 @@ +import { TxtAnswer } from 'dns-packet'; +import { + endpoints as defaultEndpoints, + Endpoint, + EndpointProps, + query, +} from 'dns-query'; + +import { DnsClient } from './dns'; + +const { cloudflare, google, opendns } = defaultEndpoints; + +export type Endpoints = + | 'doh' + | 'dns' + | Iterable; + +export class DnsOverHttps implements DnsClient { + /** + * Create new Dns-Over-Http DNS client. + * + * @param endpoints The endpoints for Dns-Over-Https queries. + * See [dns-query](https://www.npmjs.com/package/dns-query) for details. + * + * @throws {code: string} If DNS query fails. + */ + public constructor( + public endpoints: Endpoints = [cloudflare, google, opendns] + ) {} + + async resolveTXT(domain: string): Promise { + const response = await query({ + questions: [{ type: 'TXT', name: domain }], + }); + + const answers = response.answers as TxtAnswer[]; + + const data = answers.map((a) => a.data); + + const result: string[] = []; + + data.forEach((d) => { + if (typeof d === 'string') { + result.push(d); + } else if (Array.isArray(d)) { + d.forEach((sd) => { + if (typeof sd === 'string') { + result.push(sd); + } else { + result.push(Buffer.from(sd).toString('utf-8')); + } + }); + } else { + result.push(Buffer.from(d).toString('utf-8')); + } + }); + + return result; + } +} diff --git a/src/lib/discovery/enrtree.spec.ts b/src/lib/discovery/enrtree.spec.ts new file mode 100644 index 0000000000..af7402b82e --- /dev/null +++ b/src/lib/discovery/enrtree.spec.ts @@ -0,0 +1,89 @@ +import { expect } from 'chai'; + +import { ENRTree } from './enrtree'; +import testData from './testdata.json'; + +const dns = testData.dns; + +describe('ENRTree', () => { + // Root DNS entries + it('ENRTree (root): should parse and verify and DNS root entry', () => { + const subdomain = ENRTree.parseAndVerifyRoot(dns.enrRoot, dns.publicKey); + + expect(subdomain).to.eq('JORXBYVVM7AEKETX5DGXW44EAY'); + }); + + it('ENRTree (root): should error if DNS root entry is mis-prefixed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadPrefix, dns.publicKey); + } catch (e) { + expect(e.toString()).includes( + "ENRTree root entry must start with 'enrtree-root:'" + ); + } + }); + + it('ENRTree (root): should error if DNS root entry signature is invalid', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadSig, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Unable to verify ENRTree root signature'); + } + }); + + it('ENRTree (root): should error if DNS root entry is malformed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootMalformed, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree root entry'); + } + }); + + // Tree DNS entries + it('ENRTree (tree): should parse a DNS tree entry', () => { + const { publicKey, domain } = ENRTree.parseTree(dns.enrTree); + + expect(publicKey).to.eq(dns.publicKey); + expect(domain).to.eq('nodes.example.org'); + }); + + it('ENRTree (tree): should error if DNS tree entry is mis-prefixed', () => { + try { + ENRTree.parseTree(dns.enrTreeBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree tree entry must start with 'enrtree:'" + ); + } + }); + + it('ENRTree (tree): should error if DNS tree entry is misformatted', () => { + try { + ENRTree.parseTree(dns.enrTreeMalformed); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree tree entry'); + } + }); + + // Branch entries + it('ENRTree (branch): should parse and verify a single component DNS branch entry', () => { + const expected = [ + 'D2SNLTAGWNQ34NTQTPHNZDECFU', + '67BLTJEU5R2D5S3B4QKJSBRFCY', + 'A2HDMZBB4JIU53VTEGC4TG6P4A', + ]; + + const branches = ENRTree.parseBranch(dns.enrBranch); + expect(branches).to.deep.eq(expected); + }); + + it('ENRTree (branch): should error if DNS branch entry is mis-prefixed', () => { + try { + ENRTree.parseBranch(dns.enrBranchBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree branch entry must start with 'enrtree-branch:'" + ); + } + }); +}); diff --git a/src/lib/discovery/enrtree.ts b/src/lib/discovery/enrtree.ts new file mode 100644 index 0000000000..d256cab9ec --- /dev/null +++ b/src/lib/discovery/enrtree.ts @@ -0,0 +1,123 @@ +import assert from 'assert'; + +import base64url from 'base64url'; +import * as base32 from 'hi-base32'; +import { ecdsaVerify } from 'secp256k1'; + +import { ENR } from '../enr'; +import { keccak256Buf } from '../utils'; + +export interface PeerInfo { + id?: Uint8Array | Buffer; + address?: string; + udpPort?: number | null; + tcpPort?: number | null; +} + +type ENRRootValues = { + eRoot: string; + lRoot: string; + seq: number; + signature: string; +}; + +type ENRTreeValues = { + publicKey: string; + domain: string; +}; + +export class ENRTree { + public static readonly RECORD_PREFIX = ENR.RECORD_PREFIX; + public static readonly TREE_PREFIX = 'enrtree:'; + public static readonly BRANCH_PREFIX = 'enrtree-branch:'; + public static readonly ROOT_PREFIX = 'enrtree-root:'; + + /** + * Extracts the branch subdomain referenced by a DNS tree root string after verifying + * the root record signature with its base32 compressed public key. + */ + static parseAndVerifyRoot(root: string, publicKey: string): string { + assert( + root.startsWith(this.ROOT_PREFIX), + `ENRTree root entry must start with '${this.ROOT_PREFIX}'` + ); + + const rootValues = ENRTree.parseRootValues(root); + const decodedPublicKey = base32.decode.asBytes(publicKey); + + // The signature is a 65-byte secp256k1 over the keccak256 hash + // of the record content, excluding the `sig=` part, encoded as URL-safe base64 string + // (Trailing recovery bit must be trimmed to pass `ecdsaVerify` method) + const signedComponent = root.split(' sig')[0]; + const signedComponentBuffer = Buffer.from(signedComponent); + const signatureBuffer = base64url + .toBuffer(rootValues.signature) + .slice(0, 64); + const keyBuffer = Buffer.from(decodedPublicKey); + + const isVerified = ecdsaVerify( + signatureBuffer, + keccak256Buf(signedComponentBuffer), + keyBuffer + ); + + assert(isVerified, 'Unable to verify ENRTree root signature'); + + return rootValues.eRoot; + } + + static parseRootValues(txt: string): ENRRootValues { + const matches = txt.match( + /^enrtree-root:v1 e=([^ ]+) l=([^ ]+) seq=(\d+) sig=([^ ]+)$/ + ); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree root entry'); + + matches.shift(); // The first entry is the full match + const [eRoot, lRoot, seq, signature] = matches; + + assert.ok(eRoot, "Could not parse 'e' value from ENRTree root entry"); + assert.ok(lRoot, "Could not parse 'l' value from ENRTree root entry"); + assert.ok(seq, "Could not parse 'seq' value from ENRTree root entry"); + assert.ok(signature, "Could not parse 'sig' value from ENRTree root entry"); + + return { eRoot, lRoot, seq: Number(seq), signature }; + } + + /** + * Returns the public key and top level domain of an ENR tree entry. + * The domain is the starting point for traversing a set of linked DNS TXT records + * and the public key is used to verify the root entry record + */ + static parseTree(tree: string): ENRTreeValues { + assert( + tree.startsWith(this.TREE_PREFIX), + `ENRTree tree entry must start with '${this.TREE_PREFIX}'` + ); + + const matches = tree.match(/^enrtree:\/\/([^@]+)@(.+)$/); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree tree entry'); + + matches.shift(); // The first entry is the full match + const [publicKey, domain] = matches; + + assert.ok(publicKey, 'Could not parse public key from ENRTree tree entry'); + assert.ok(domain, 'Could not parse domain from ENRTree tree entry'); + + return { publicKey, domain }; + } + + /** + * Returns subdomains listed in an ENR branch entry. These in turn lead to + * either further branch entries or ENR records. + */ + static parseBranch(branch: string): string[] { + assert( + branch.startsWith(this.BRANCH_PREFIX), + `ENRTree branch entry must start with '${this.BRANCH_PREFIX}'` + ); + + return branch.split(this.BRANCH_PREFIX)[1].split(','); + } +} diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index 09eeb32682..5513e332c0 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -1,6 +1,7 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; +export { parseBootstrap } from './bootstrap'; export function getPseudoRandomSubset( values: string[], diff --git a/src/lib/discovery/testdata.json b/src/lib/discovery/testdata.json new file mode 100644 index 0000000000..86a3a7b1be --- /dev/null +++ b/src/lib/discovery/testdata.json @@ -0,0 +1,18 @@ +{ + "dns": { + "publicKey": "AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE", + "enr": "enr:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrA": "enr:-Jq4QAopXcF_SSfOwl_AmLdrMUnHQO1Rx-XV4gYeySSK32PTbQ8volkh3IQy1ag1Gkl6O-C5rjskj3EyDi8XVzck4PMVg2V0aMrJhKALwySDbxWAgmlkgnY0gmlwhC1NKH-Jc2VjcDI1NmsxoQO5wMEjJLtqT-h6zhef0xsO-SW-pcQD-yuNqCr3GTEZFoN0Y3CCdl-DdWRwgnZf", + "enrB": "enr:-Je4QAFx_6rFjCxCLPUbxIA_KS7FhCYeTU6fXmbj1V08f8DPCUAB9bLoY2Yy7q2hIEby7Yf6e_v7gbofloB1oTnjqeYDg2V0aMfGhOAp6ZGAgmlkgnY0gmlwhLxf-D2Jc2VjcDI1NmsxoQOou7vgUXL96E5CzBsCE6N1GSMqlAACtUxRiNpq6vnB6IN0Y3CCdl-DdWRwgnZf", + "enrRoot": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranch": "enrtree-branch:D2SNLTAGWNQ34NTQTPHNZDECFU,67BLTJEU5R2D5S3B4QKJSBRFCY,A2HDMZBB4JIU53VTEGC4TG6P4A", + "enrTree": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@nodes.example.org", + "enrBadPrefix": "enrabc:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrRootBadPrefix": "enrtree:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranchBadPrefix": "Z64M,JOECK7UUYUFVX24QGXYLR3UHDU,RR6SC4GUZBKLFA2WO4IUY6YGEE,EQRME5EAOS7AJHHLDDZNDYT7GI,JXHUMLDSGKU6UQWYFMNCFYQFHQ,4SNDLPNM3CBG2KLBMRSTHWFNP4,WEEEFCKUXOGU4QPKCRBBEHQLEY,CPXM5AOSTICZ3TODJFQACGBWMU,7U26GD37NS6DV72PDAURZI4WUY,MYLQIGMR5GTKPPBMXIINZ2ALGU", + "enrTreeBadPrefix": "entree-branch://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@nodes.example.org", + "enrRootBadSig": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Aa7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrRootMalformed": "enrtree-root:v1 e=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrTreeMalformed": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2nodes.example.org" + } +} diff --git a/src/lib/enr/enr.spec.ts b/src/lib/enr/enr.spec.ts index e9e4d133c5..c7feb48610 100644 --- a/src/lib/enr/enr.spec.ts +++ b/src/lib/enr/enr.spec.ts @@ -113,7 +113,7 @@ describe('ENR', function () { ENR.decodeTxt(txt); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); }); @@ -153,7 +153,7 @@ describe('ENR', function () { enr.verify(Buffer.alloc(0), Buffer.alloc(0)); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); diff --git a/src/lib/enr/enr.ts b/src/lib/enr/enr.ts index 440ca16ba2..d41e6584b2 100644 --- a/src/lib/enr/enr.ts +++ b/src/lib/enr/enr.ts @@ -25,6 +25,7 @@ import { ENRKey, ENRValue, NodeId, SequenceNumber } from './types'; import * as v4 from './v4'; export class ENR extends Map { + public static readonly RECORD_PREFIX = 'enr:'; public seq: SequenceNumber; public signature: Buffer | null; @@ -93,8 +94,10 @@ export class ENR extends Map { } static decodeTxt(encoded: string): ENR { - if (!encoded.startsWith('enr:')) { - throw new Error("string encoded ENR must start with 'enr:'"); + if (!encoded.startsWith(this.RECORD_PREFIX)) { + throw new Error( + `"string encoded ENR must start with '${this.RECORD_PREFIX}'` + ); } return ENR.decode(base64url.toBuffer(encoded.slice(4))); } @@ -388,6 +391,7 @@ export class ENR extends Map { return new Multiaddr(maBuf); } + setLocationMultiaddr(multiaddr: Multiaddr): void { const protoNames = multiaddr.protoNames(); if ( @@ -428,7 +432,7 @@ export class ENR extends Map { throw new Error(ERR_INVALID_ID); } if (!this.publicKey) { - throw new Error('Failed to verify enr: No public key'); + throw new Error('Failed to verify ENR: No public key'); } return v4.verify(this.publicKey, data, signature); } @@ -471,6 +475,6 @@ export class ENR extends Map { } encodeTxt(privateKey?: Buffer): string { - return 'enr:' + base64url.encode(this.encode(privateKey)); + return ENR.RECORD_PREFIX + base64url.encode(this.encode(privateKey)); } } diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 605e5b95dc..398e450c31 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,3 +1,5 @@ +import { keccak256, Message } from 'js-sha3'; + export function hexToBuf(hex: string | Buffer | Uint8Array): Buffer { if (typeof hex === 'string') { return Buffer.from(hex.replace(/^0x/i, ''), 'hex'); @@ -31,3 +33,7 @@ export function equalByteArrays( return aBuf.compare(bBuf) === 0; } + +export function keccak256Buf(message: Message): Buffer { + return Buffer.from(keccak256.arrayBuffer(message)); +} diff --git a/src/lib/waku.ts b/src/lib/waku.ts index f5da1c5d3e..de942c89e7 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -18,7 +18,7 @@ import Ping from 'libp2p/src/ping'; import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; -import { getNodesFromHostedJson } from './discovery'; +import { parseBootstrap } from './discovery'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; import { WakuMessage } from './waku_message'; @@ -86,10 +86,7 @@ export interface CreateOptions { /** * Use libp2p-bootstrap to discover and connect to new nodes. * - * You can pass: - * - `true` to use {@link getNodesFromHostedJson}, - * - an array of multiaddresses, - * - a function that returns an array of multiaddresses (or Promise of). + * See [BootstrapOptions] for available parameters. * * Note: It overrides any other peerDiscovery modules that may have been set via * {@link CreateOptions.libp2p}. @@ -189,17 +186,7 @@ export class Waku { }); if (options?.bootstrap) { - let bootstrap: undefined | (() => string[] | Promise); - - if (options.bootstrap === true) { - bootstrap = getNodesFromHostedJson; - } else if (Array.isArray(options.bootstrap)) { - bootstrap = (): string[] => { - return options.bootstrap as string[]; - }; - } else if (typeof options.bootstrap === 'function') { - bootstrap = options.bootstrap; - } + const bootstrap = parseBootstrap(options?.bootstrap); if (bootstrap !== undefined) { try { From 284644b8221f4269495e3bb157698278f9047c06 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 14:28:45 +1100 Subject: [PATCH 04/16] Fix-up new bootstrap API --- CHANGELOG.md | 2 + examples/eth-pm-wallet-encryption/src/waku.ts | 2 +- examples/eth-pm/src/waku.ts | 2 +- examples/web-chat/src/App.tsx | 11 +- src/lib/discovery/bootstrap.ts | 116 ++++++++---------- src/lib/discovery/hosted_json.ts | 6 +- src/lib/discovery/index.ts | 6 +- src/lib/waku.node.spec.ts | 10 +- src/lib/waku.ts | 5 +- 9 files changed, 80 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8901263351..4d7f5f6a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Test: Upgrade nim-waku node to v0.6. - **Breaking**: Renamed `getBootstrapNodes` to `getNodesFromHostedJson`. - Minimum node version changed to 16. +- **Breaking**: Changed `Waku.create` bootstrap option from `{ bootstrap: boolean }` to `{ bootstrap: BootstrapOptions }`. + Replace `{ boostrap: true }` with `{ boostrap: { default: true } }` to retain same behaviour. ### Fixed diff --git a/examples/eth-pm-wallet-encryption/src/waku.ts b/examples/eth-pm-wallet-encryption/src/waku.ts index d0be5baf83..f808dff396 100644 --- a/examples/eth-pm-wallet-encryption/src/waku.ts +++ b/examples/eth-pm-wallet-encryption/src/waku.ts @@ -11,7 +11,7 @@ export const PrivateMessageContentTopic = '/eth-pm-wallet/1/private-message/proto'; export async function initWaku(): Promise { - const waku = await Waku.create({ bootstrap: true }); + const waku = await Waku.create({ bootstrap: { default: true } }); // Wait to be connected to at least one peer await new Promise((resolve, reject) => { diff --git a/examples/eth-pm/src/waku.ts b/examples/eth-pm/src/waku.ts index 8a3ba14c94..33e802bcef 100644 --- a/examples/eth-pm/src/waku.ts +++ b/examples/eth-pm/src/waku.ts @@ -9,7 +9,7 @@ export const PublicKeyContentTopic = '/eth-pm/1/public-key/proto'; export const PrivateMessageContentTopic = '/eth-pm/1/private-message/proto'; export async function initWaku(): Promise { - const waku = await Waku.create({ bootstrap: true }); + const waku = await Waku.create({ bootstrap: { default: true } }); // Wait to be connected to at least one peer await new Promise((resolve, reject) => { diff --git a/examples/web-chat/src/App.tsx b/examples/web-chat/src/App.tsx index ec42eda8eb..02a81caf75 100644 --- a/examples/web-chat/src/App.tsx +++ b/examples/web-chat/src/App.tsx @@ -1,6 +1,11 @@ import { useEffect, useReducer, useState } from 'react'; import './App.css'; -import { PageDirection, getBootstrapNodes, Waku, WakuMessage } from 'js-waku'; +import { + PageDirection, + getNodesFromHostedJson, + Waku, + WakuMessage, +} from 'js-waku'; import handleCommand from './command'; import Room from './Room'; import { WakuContext } from './WakuContext'; @@ -175,7 +180,9 @@ async function initWaku(setter: (waku: Waku) => void) { }, }, }, - bootstrap: getBootstrapNodes.bind({}, selectFleetEnv()), + bootstrap: { + getPeers: getNodesFromHostedJson.bind({}, selectFleetEnv()), + }, }); setter(waku); diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts index 6ba0860bc0..14a004edb7 100644 --- a/src/lib/discovery/bootstrap.ts +++ b/src/lib/discovery/bootstrap.ts @@ -1,4 +1,5 @@ import debug from 'debug'; +import { Multiaddr } from 'multiaddr'; import { DnsNodeDiscovery } from './dns'; @@ -6,7 +7,9 @@ import { getNodesFromHostedJson, getPseudoRandomSubset } from './index'; const dbg = debug('waku:discovery:bootstrap'); -const DefaultMaxPeers = 1; +export const DefaultMaxPeers = 1; + +export type BootstrapFn = () => Promise; /** * Setup discovery method used to bootstrap. @@ -17,11 +20,11 @@ export interface BootstrapOptions { /** * The maximum of peers to connect to as part of the bootstrap process. * - * @default 1 + * @default {{DefaultMaxPeers}} */ maxPeers?: number; /** - * Use the default discovery method. + * Use the default discovery method. Overrides all other options but `maxPeers` * * The default discovery method is likely to change overtime as new discovery * methods are implemented. @@ -36,7 +39,7 @@ export interface BootstrapOptions { /** * Getter that retrieve multiaddrs of peers to connect to. */ - getPeers?: () => Promise; + getPeers?: () => Promise; /** * An EIP-1459 ENR Tree URL. For example: * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev" @@ -47,69 +50,54 @@ export interface BootstrapOptions { /** * Parse the bootstrap options and returns an async function that returns node addresses upon invocation. */ -export function parseBootstrap( - options: - | BootstrapOptions - | boolean - | string[] - | (() => string[] | Promise) -): undefined | (() => Promise) { - if ( - Object.keys(options).includes('default') || - Object.keys(options).includes('maxPeers') || - Object.keys(options).includes('peers') || - Object.keys(options).includes('getPeers') || - Object.keys(options).includes('enrUrl') - ) { - const opts = options as unknown as BootstrapOptions; - const maxPeers = opts.maxPeers || DefaultMaxPeers; +export function parseBootstrap(opts: BootstrapOptions): BootstrapFn { + const maxPeers = opts.maxPeers ?? DefaultMaxPeers; - if (opts.default) { - dbg('Bootstrap: Use hosted list of peers.'); + if (opts.default) { + dbg('Bootstrap: Use hosted list of peers.'); - return getNodesFromHostedJson.bind({}, undefined, undefined, maxPeers); - } else if (opts.peers !== undefined && opts.peers.length > 0) { - dbg('Bootstrap: Use provided list of peers.'); + return getNodesFromHostedJson.bind({}, undefined, undefined, maxPeers); + } else if (opts.peers !== undefined && opts.peers.length > 0) { + dbg('Bootstrap: Use provided list of peers.'); - const allPeers: string[] = opts.peers; - return (): Promise => { - const peers = getPseudoRandomSubset(allPeers, maxPeers); - return Promise.resolve(peers); - }; - } else if (typeof opts.getPeers === 'function') { - dbg('Bootstrap: Use provided getPeers function.'); - const getPeers = opts.getPeers; - - return async (): Promise => { - const allPeers = await getPeers(); - return getPseudoRandomSubset(allPeers, maxPeers); - }; - } else if (opts.enrUrl) { - const enrUrl = opts.enrUrl; - dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); - - const dns = DnsNodeDiscovery.dnsOverHttp(); - // TODO: The `toString` is incorrect. - return (): Promise => - dns - .getPeers(maxPeers, [enrUrl]) - .then((peers) => peers.map((peer) => peer.toString())); - } - } else { - dbg( - 'WARN: This bootstrap method will be deprecated, use `BootstrapOptions` instead' + const allPeers: Multiaddr[] = opts.peers.map( + (node: string) => new Multiaddr(node) ); - if (options === true) { - return getNodesFromHostedJson; - } else if (Array.isArray(options)) { - return (): Promise => { - return Promise.resolve(options as string[]); - }; - } else if (typeof options === 'function') { - return async (): Promise => { - return options(); - }; - } + const peers = getPseudoRandomSubset(allPeers, maxPeers); + return (): Promise => Promise.resolve(peers); + } else if (typeof opts.getPeers === 'function') { + dbg('Bootstrap: Use provided getPeers function.'); + const getPeers = opts.getPeers; + + return async (): Promise => { + const allPeers = await getPeers(); + return getPseudoRandomSubset(allPeers, maxPeers).map( + (node) => new Multiaddr(node) + ); + }; + } else if (opts.enrUrl) { + const enrUrl = opts.enrUrl; + dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); + + const dns = DnsNodeDiscovery.dnsOverHttp(); + + return async (): Promise => { + const enrs = await dns.getPeers(maxPeers, [enrUrl]); + const addresses: Multiaddr[] = []; + enrs.forEach((enr) => { + if (!enr.multiaddrs) return; + + enr.multiaddrs.forEach((ma: Multiaddr) => { + // Only return secure websocket addresses + if (ma.protoNames().includes('wss')) { + addresses.push(ma); + } + }); + }); + return addresses; + }; + } else { + dbg('No bootstrap method specified, no peer will be returned'); + return (): Promise => Promise.resolve([]); } - return; } diff --git a/src/lib/discovery/hosted_json.ts b/src/lib/discovery/hosted_json.ts index bfabea3aca..111675e20d 100644 --- a/src/lib/discovery/hosted_json.ts +++ b/src/lib/discovery/hosted_json.ts @@ -28,7 +28,7 @@ export async function getNodesFromHostedJson( path: string[] = ['fleets', 'wakuv2.prod', 'waku-websocket'], url = 'https://fleets.status.im/', wantedNumber: number = DefaultWantedNumber -): Promise { +): Promise { if (wantedNumber <= 0) { return []; } @@ -54,12 +54,12 @@ export async function getNodesFromHostedJson( if (Array.isArray(nodes)) { return getPseudoRandomSubset(nodes, wantedNumber).map( - (node: string) => node + (node: string) => new Multiaddr(node) ); } if (typeof nodes === 'string') { - return [nodes]; + return [new Multiaddr(nodes)]; } if (typeof nodes === 'object') { diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index 5513e332c0..e3c9b429d4 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -3,10 +3,10 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; export { parseBootstrap } from './bootstrap'; -export function getPseudoRandomSubset( - values: string[], +export function getPseudoRandomSubset( + values: T[], wantedNumber: number -): string[] { +): T[] { if (values.length <= wantedNumber) { return values; } diff --git a/src/lib/waku.node.spec.ts b/src/lib/waku.node.spec.ts index ce50591170..4e2a3cffd5 100644 --- a/src/lib/waku.node.spec.ts +++ b/src/lib/waku.node.spec.ts @@ -38,7 +38,7 @@ describe('Waku Dial [node only]', function () { waku = await Waku.create({ staticNoiseKey: NOISE_KEY_1, - bootstrap: true, + bootstrap: { default: true }, }); const connectedPeerID: PeerId = await new Promise((resolve) => { @@ -68,7 +68,7 @@ describe('Waku Dial [node only]', function () { libp2p: { modules: { transport: [TCP] }, }, - bootstrap: [multiAddrWithId], + bootstrap: { peers: [multiAddrWithId] }, }); const connectedPeerID: PeerId = await new Promise((resolve) => { @@ -102,8 +102,10 @@ describe('Waku Dial [node only]', function () { libp2p: { modules: { transport: [TCP] }, }, - bootstrap: () => { - return [multiAddrWithId]; + bootstrap: { + getPeers: async () => { + return [multiAddrWithId]; + }, }, }); diff --git a/src/lib/waku.ts b/src/lib/waku.ts index de942c89e7..f2cf1f85e8 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -19,6 +19,7 @@ import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; import { parseBootstrap } from './discovery'; +import { BootstrapOptions } from './discovery/bootstrap'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; import { WakuMessage } from './waku_message'; @@ -86,12 +87,12 @@ export interface CreateOptions { /** * Use libp2p-bootstrap to discover and connect to new nodes. * - * See [BootstrapOptions] for available parameters. + * See [[BootstrapOptions]] for available parameters. * * Note: It overrides any other peerDiscovery modules that may have been set via * {@link CreateOptions.libp2p}. */ - bootstrap?: boolean | string[] | (() => string[] | Promise); + bootstrap?: BootstrapOptions; decryptionKeys?: Array; } From 4bfe060064422a6a307297ba62f2b6b87865fe00 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 15:42:25 +1100 Subject: [PATCH 05/16] Export BootstrapOptions --- src/index.ts | 8 +++++++- src/lib/discovery/index.ts | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 4887685ba3..2b318f1f5b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,10 @@ -export { getNodesFromHostedJson, getPseudoRandomSubset } from './lib/discovery'; +export { + getNodesFromHostedJson, + getPseudoRandomSubset, + parseBootstrap, + BootstrapOptions, + BootstrapFn, +} from './lib/discovery'; export * as utils from './lib/utils'; diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index e3c9b429d4..dfc4be4c4f 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -1,7 +1,7 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; -export { parseBootstrap } from './bootstrap'; +export { parseBootstrap, BootstrapOptions, BootstrapFn } from './bootstrap'; export function getPseudoRandomSubset( values: T[], From e871da056ff014724fcf6e99efe589b596cd8f93 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 15:53:22 +1100 Subject: [PATCH 06/16] Export enr, dns, entree modules --- src/index.ts | 11 ++++------- src/lib/discovery/bootstrap.ts | 2 +- src/lib/discovery/enrtree.ts | 7 ------- src/lib/discovery/index.ts | 3 +++ 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/index.ts b/src/index.ts index 2b318f1f5b..d3a72319e6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,7 @@ -export { - getNodesFromHostedJson, - getPseudoRandomSubset, - parseBootstrap, - BootstrapOptions, - BootstrapFn, -} from './lib/discovery'; +export { getNodesFromHostedJson } from './lib/discovery'; +export * as discovery from './lib/discovery'; + +export * as enr from './lib/enr'; export * as utils from './lib/utils'; diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts index 14a004edb7..fda17a6f80 100644 --- a/src/lib/discovery/bootstrap.ts +++ b/src/lib/discovery/bootstrap.ts @@ -20,7 +20,7 @@ export interface BootstrapOptions { /** * The maximum of peers to connect to as part of the bootstrap process. * - * @default {{DefaultMaxPeers}} + * @default [[DefaultMaxPeers]] */ maxPeers?: number; /** diff --git a/src/lib/discovery/enrtree.ts b/src/lib/discovery/enrtree.ts index d256cab9ec..369b294c02 100644 --- a/src/lib/discovery/enrtree.ts +++ b/src/lib/discovery/enrtree.ts @@ -7,13 +7,6 @@ import { ecdsaVerify } from 'secp256k1'; import { ENR } from '../enr'; import { keccak256Buf } from '../utils'; -export interface PeerInfo { - id?: Uint8Array | Buffer; - address?: string; - udpPort?: number | null; - tcpPort?: number | null; -} - type ENRRootValues = { eRoot: string; lRoot: string; diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index dfc4be4c4f..b0e04fd44d 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -2,6 +2,9 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; export { parseBootstrap, BootstrapOptions, BootstrapFn } from './bootstrap'; +export { DnsClient, DnsNodeDiscovery } from './dns'; +export { Endpoints, DnsOverHttps } from './dns_over_https'; +export { ENRTree } from './enrtree'; export function getPseudoRandomSubset( values: T[], From f0eb9d1609ba5f6bfbe044f4f4fb431813663ee6 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:03:07 +1100 Subject: [PATCH 07/16] Rename libp2p bootstrap import To avoid name conflict. --- src/lib/waku.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/waku.ts b/src/lib/waku.ts index f2cf1f85e8..aaa83636e5 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -2,7 +2,7 @@ import { bytes } from '@chainsafe/libp2p-noise/dist/src/@types/basic'; import { Noise } from '@chainsafe/libp2p-noise/dist/src/noise'; import debug from 'debug'; import Libp2p, { Connection, Libp2pModules, Libp2pOptions } from 'libp2p'; -import Bootstrap from 'libp2p-bootstrap'; +import Libp2pBootstrap from 'libp2p-bootstrap'; import { MuxedStream } from 'libp2p-interfaces/dist/src/stream-muxer/types'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: No types available @@ -195,11 +195,11 @@ export class Waku { // Note: this overrides any other peer discover libp2pOpts.modules = Object.assign(libp2pOpts.modules, { - peerDiscovery: [Bootstrap], + peerDiscovery: [Libp2pBootstrap], }); libp2pOpts.config.peerDiscovery = { - [Bootstrap.tag]: { + [Libp2pBootstrap.tag]: { list, enabled: true, }, From ae0faa2146ef0a41bca2aeced25f48ba4788dcde Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:04:07 +1100 Subject: [PATCH 08/16] Clarify default DNS servers --- src/lib/discovery/dns_over_https.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/discovery/dns_over_https.ts b/src/lib/discovery/dns_over_https.ts index d71fb80130..9f0b33e70f 100644 --- a/src/lib/discovery/dns_over_https.ts +++ b/src/lib/discovery/dns_over_https.ts @@ -21,6 +21,7 @@ export class DnsOverHttps implements DnsClient { * * @param endpoints The endpoints for Dns-Over-Https queries. * See [dns-query](https://www.npmjs.com/package/dns-query) for details. + * Defaults to cloudflare, google and opendns. * * @throws {code: string} If DNS query fails. */ From fa4e94750f503999458f26f61346149ff1cf9bb6 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:04:57 +1100 Subject: [PATCH 09/16] Make parseBootstrap a class Makes it easier to use the resulting function and document defaults. --- src/lib/discovery/bootstrap.ts | 101 ++++++++++++++++++--------------- src/lib/discovery/index.ts | 2 +- src/lib/waku.ts | 8 +-- 3 files changed, 60 insertions(+), 51 deletions(-) diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts index fda17a6f80..771eaad558 100644 --- a/src/lib/discovery/bootstrap.ts +++ b/src/lib/discovery/bootstrap.ts @@ -7,10 +7,6 @@ import { getNodesFromHostedJson, getPseudoRandomSubset } from './index'; const dbg = debug('waku:discovery:bootstrap'); -export const DefaultMaxPeers = 1; - -export type BootstrapFn = () => Promise; - /** * Setup discovery method used to bootstrap. * @@ -20,7 +16,7 @@ export interface BootstrapOptions { /** * The maximum of peers to connect to as part of the bootstrap process. * - * @default [[DefaultMaxPeers]] + * @default [[Bootstrap.DefaultMaxPeers]] */ maxPeers?: number; /** @@ -48,56 +44,69 @@ export interface BootstrapOptions { } /** - * Parse the bootstrap options and returns an async function that returns node addresses upon invocation. + * Parse options and expose function to return bootstrap peer addresses. */ -export function parseBootstrap(opts: BootstrapOptions): BootstrapFn { - const maxPeers = opts.maxPeers ?? DefaultMaxPeers; +export class Bootstrap { + public static DefaultMaxPeers = 1; - if (opts.default) { - dbg('Bootstrap: Use hosted list of peers.'); + public readonly getBootstrapPeers: (() => Promise) | undefined; - return getNodesFromHostedJson.bind({}, undefined, undefined, maxPeers); - } else if (opts.peers !== undefined && opts.peers.length > 0) { - dbg('Bootstrap: Use provided list of peers.'); + constructor(opts: BootstrapOptions) { + const maxPeers = opts.maxPeers ?? Bootstrap.DefaultMaxPeers; - const allPeers: Multiaddr[] = opts.peers.map( - (node: string) => new Multiaddr(node) - ); - const peers = getPseudoRandomSubset(allPeers, maxPeers); - return (): Promise => Promise.resolve(peers); - } else if (typeof opts.getPeers === 'function') { - dbg('Bootstrap: Use provided getPeers function.'); - const getPeers = opts.getPeers; + if (opts.default) { + dbg('Bootstrap: Use hosted list of peers.'); - return async (): Promise => { - const allPeers = await getPeers(); - return getPseudoRandomSubset(allPeers, maxPeers).map( - (node) => new Multiaddr(node) + this.getBootstrapPeers = getNodesFromHostedJson.bind( + {}, + undefined, + undefined, + maxPeers ); - }; - } else if (opts.enrUrl) { - const enrUrl = opts.enrUrl; - dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); + } else if (opts.peers !== undefined && opts.peers.length > 0) { + dbg('Bootstrap: Use provided list of peers.'); - const dns = DnsNodeDiscovery.dnsOverHttp(); + const allPeers: Multiaddr[] = opts.peers.map( + (node: string) => new Multiaddr(node) + ); + const peers = getPseudoRandomSubset(allPeers, maxPeers); + this.getBootstrapPeers = (): Promise => + Promise.resolve(peers); + } else if (typeof opts.getPeers === 'function') { + dbg('Bootstrap: Use provided getPeers function.'); + const getPeers = opts.getPeers; - return async (): Promise => { - const enrs = await dns.getPeers(maxPeers, [enrUrl]); - const addresses: Multiaddr[] = []; - enrs.forEach((enr) => { - if (!enr.multiaddrs) return; + this.getBootstrapPeers = async (): Promise => { + const allPeers = await getPeers(); + return getPseudoRandomSubset( + allPeers, + maxPeers + ).map((node) => new Multiaddr(node)); + }; + } else if (opts.enrUrl) { + const enrUrl = opts.enrUrl; + dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); - enr.multiaddrs.forEach((ma: Multiaddr) => { - // Only return secure websocket addresses - if (ma.protoNames().includes('wss')) { - addresses.push(ma); - } + const dns = DnsNodeDiscovery.dnsOverHttp(); + + this.getBootstrapPeers = async (): Promise => { + const enrs = await dns.getPeers(maxPeers, [enrUrl]); + const addresses: Multiaddr[] = []; + enrs.forEach((enr) => { + if (!enr.multiaddrs) return; + + enr.multiaddrs.forEach((ma: Multiaddr) => { + // Only return secure websocket addresses + if (ma.protoNames().includes('wss')) { + addresses.push(ma); + } + }); }); - }); - return addresses; - }; - } else { - dbg('No bootstrap method specified, no peer will be returned'); - return (): Promise => Promise.resolve([]); + return addresses; + }; + } else { + dbg('No bootstrap method specified, no peer will be returned'); + this.getBootstrapPeers = undefined; + } } } diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index b0e04fd44d..d18088a0d4 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -1,7 +1,7 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; -export { parseBootstrap, BootstrapOptions, BootstrapFn } from './bootstrap'; +export { Bootstrap, BootstrapOptions } from './bootstrap'; export { DnsClient, DnsNodeDiscovery } from './dns'; export { Endpoints, DnsOverHttps } from './dns_over_https'; export { ENRTree } from './enrtree'; diff --git a/src/lib/waku.ts b/src/lib/waku.ts index aaa83636e5..977b07c19b 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -18,7 +18,7 @@ import Ping from 'libp2p/src/ping'; import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; -import { parseBootstrap } from './discovery'; +import { Bootstrap } from './discovery'; import { BootstrapOptions } from './discovery/bootstrap'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; @@ -187,11 +187,11 @@ export class Waku { }); if (options?.bootstrap) { - const bootstrap = parseBootstrap(options?.bootstrap); + const bootstrap = new Bootstrap(options?.bootstrap); - if (bootstrap !== undefined) { + if (bootstrap.getBootstrapPeers !== undefined) { try { - const list = await bootstrap(); + const list = await bootstrap.getBootstrapPeers(); // Note: this overrides any other peer discover libp2pOpts.modules = Object.assign(libp2pOpts.modules, { From 6276b1537f5fdf63175faacbd4edad98179e3ee2 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:05:44 +1100 Subject: [PATCH 10/16] Remove unnecessary declaration of Buffer It's handled by the polyfill config in webpack. --- src/lib/enr/v4.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/enr/v4.ts b/src/lib/enr/v4.ts index eae00be7f0..7024541b75 100644 --- a/src/lib/enr/v4.ts +++ b/src/lib/enr/v4.ts @@ -1,4 +1,3 @@ -import { Buffer } from 'buffer'; import crypto from 'crypto'; import { keccak256 } from 'js-sha3'; From b65ab17cb935305d862035c31a22e21da553e54f Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:09:01 +1100 Subject: [PATCH 11/16] Expose types to ensure documentation is complete --- src/lib/discovery/dns.ts | 2 +- src/lib/discovery/enrtree.ts | 4 ++-- src/lib/discovery/index.ts | 4 ++-- src/lib/enr/index.ts | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/discovery/dns.ts b/src/lib/discovery/dns.ts index ba3efd00f6..34f4b2b10a 100644 --- a/src/lib/discovery/dns.ts +++ b/src/lib/discovery/dns.ts @@ -9,7 +9,7 @@ import { ENRTree } from './enrtree'; const dbg = debug('waku:discovery:dns'); -type SearchContext = { +export type SearchContext = { domain: string; publicKey: string; visits: { [key: string]: boolean }; diff --git a/src/lib/discovery/enrtree.ts b/src/lib/discovery/enrtree.ts index 369b294c02..718dad922b 100644 --- a/src/lib/discovery/enrtree.ts +++ b/src/lib/discovery/enrtree.ts @@ -7,14 +7,14 @@ import { ecdsaVerify } from 'secp256k1'; import { ENR } from '../enr'; import { keccak256Buf } from '../utils'; -type ENRRootValues = { +export type ENRRootValues = { eRoot: string; lRoot: string; seq: number; signature: string; }; -type ENRTreeValues = { +export type ENRTreeValues = { publicKey: string; domain: string; }; diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index d18088a0d4..5ca9251fa9 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -2,9 +2,9 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; export { Bootstrap, BootstrapOptions } from './bootstrap'; -export { DnsClient, DnsNodeDiscovery } from './dns'; +export { DnsClient, DnsNodeDiscovery, SearchContext } from './dns'; export { Endpoints, DnsOverHttps } from './dns_over_https'; -export { ENRTree } from './enrtree'; +export { ENRTree, ENRTreeValues, ENRRootValues } from './enrtree'; export function getPseudoRandomSubset( values: T[], diff --git a/src/lib/enr/index.ts b/src/lib/enr/index.ts index 79f8330afc..8dc9b8c077 100644 --- a/src/lib/enr/index.ts +++ b/src/lib/enr/index.ts @@ -4,3 +4,4 @@ export * from './constants'; export * from './enr'; export * from './types'; export * from './create'; +export * from './keypair'; From 4fd2db608b51568b89d03826701f57d881e89967 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:18:45 +1100 Subject: [PATCH 12/16] Update caniuse in all examples --- examples/eth-pm-wallet-encryption/package-lock.json | 12 ++++++------ examples/eth-pm/package-lock.json | 12 ++++++------ examples/relay-reactjs-chat/package-lock.json | 12 ++++++------ examples/store-reactjs-chat/package-lock.json | 12 ++++++------ examples/web-chat/package-lock.json | 12 ++++++------ package-lock.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/examples/eth-pm-wallet-encryption/package-lock.json b/examples/eth-pm-wallet-encryption/package-lock.json index aa98ab43c8..9fab46b77e 100644 --- a/examples/eth-pm-wallet-encryption/package-lock.json +++ b/examples/eth-pm-wallet-encryption/package-lock.json @@ -6440,9 +6440,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27967,9 +27967,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", diff --git a/examples/eth-pm/package-lock.json b/examples/eth-pm/package-lock.json index 209e3f9b93..8744098128 100644 --- a/examples/eth-pm/package-lock.json +++ b/examples/eth-pm/package-lock.json @@ -6440,9 +6440,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27967,9 +27967,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", diff --git a/examples/relay-reactjs-chat/package-lock.json b/examples/relay-reactjs-chat/package-lock.json index 2a81ff4fc8..ccb965510f 100644 --- a/examples/relay-reactjs-chat/package-lock.json +++ b/examples/relay-reactjs-chat/package-lock.json @@ -4858,9 +4858,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -20062,9 +20062,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/examples/store-reactjs-chat/package-lock.json b/examples/store-reactjs-chat/package-lock.json index 0825849f05..4e4ad13e74 100644 --- a/examples/store-reactjs-chat/package-lock.json +++ b/examples/store-reactjs-chat/package-lock.json @@ -5383,9 +5383,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001298", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001298.tgz", - "integrity": "sha512-AcKqikjMLlvghZL/vfTHorlQsLDhGRalYf1+GmWCf5SCMziSGjRYQW/JEksj14NaYHIR6KIhrFAy0HV5C25UzQ==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -22888,9 +22888,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001298", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001298.tgz", - "integrity": "sha512-AcKqikjMLlvghZL/vfTHorlQsLDhGRalYf1+GmWCf5SCMziSGjRYQW/JEksj14NaYHIR6KIhrFAy0HV5C25UzQ==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/examples/web-chat/package-lock.json b/examples/web-chat/package-lock.json index dfb1a40e9d..7269753b47 100644 --- a/examples/web-chat/package-lock.json +++ b/examples/web-chat/package-lock.json @@ -6963,9 +6963,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27344,9 +27344,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/package-lock.json b/package-lock.json index aa3d8afbe0..f06589fdd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4879,9 +4879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001251", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001251.tgz", - "integrity": "sha512-HOe1r+9VkU4TFmnU70z+r7OLmtR+/chB1rdcJUeQlAinjEeb0cKL20tlAtOagNZhbrtLnCvV19B4FmF1rgzl6A==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -21711,9 +21711,9 @@ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, "caniuse-lite": { - "version": "1.0.30001251", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001251.tgz", - "integrity": "sha512-HOe1r+9VkU4TFmnU70z+r7OLmtR+/chB1rdcJUeQlAinjEeb0cKL20tlAtOagNZhbrtLnCvV19B4FmF1rgzl6A==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", From 0bfe9c9a648c896a7b5ab7e2c48c9ece2f29389e Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Thu, 13 Jan 2022 16:27:35 +1100 Subject: [PATCH 13/16] Do not run command on js-waku lib when running examples scripts This was happening when `d=./examples/README.md`. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 5686982d5d..9e9495390e 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "pretest": "run-s pretest:*", "pretest:1-init-git-submodules": "[ -f './nim-waku/build/wakunode2' ] || git submodule update --init --recursive", "pretest:2-build-nim-waku": "[ -f './nim-waku/build/wakunode2' ] || run-s nim-waku:build", - "examples:pretest": "for d in examples/*; do (cd $d; npm install); done", + "examples:pretest": "for d in examples/*/; do (cd $d && npm install); done", "nim-waku:build": "(cd nim-waku; NIMFLAGS=\"-d:chronicles_colors=off -d:chronicles_sinks=textlines -d:chronicles_log_level=TRACE\" make -j$(nproc --all 2>/dev/null || echo 2) wakunode2)", "nim-waku:force-build": "(cd nim-waku && rm -rf ./build/ ./vendor && make -j$(nproc --all 2>/dev/null || echo 2) update) && run-s nim-waku:build", "test": "run-s build test:*", @@ -39,7 +39,7 @@ "test:spelling": "cspell \"{README.md,.github/*.md,guides/*.md,src/**/*.ts}\"", "test:unit": "nyc --silent mocha", "test:karma": "karma start", - "examples:test": "run-s examples:pretest; for d in examples/*; do (cd $d; npm test;); done", + "examples:test": "run-s examples:pretest; for d in examples/*/; do (cd $d && npm test;); done", "proto": "run-s proto:*", "proto:lint": "buf lint", "proto:build": "buf generate", From 2c16f0befb11e1f27531e0320955eb09a90cc3ca Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Fri, 14 Jan 2022 12:44:06 +1100 Subject: [PATCH 14/16] Separate tests that use DNS data in CI --- .github/workflows/ci.yml | 39 ++++++++++++++++++++++++++++++ karma-live-data.conf.js | 45 +++++++++++++++++++++++++++++++++++ package.json | 1 + src/lib/discovery/dns.spec.ts | 6 +++++ 4 files changed, 91 insertions(+) create mode 100644 karma-live-data.conf.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b6d8049f1..11ab36c6e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,3 +90,42 @@ jobs: with: name: nim-waku-logs path: log/ + + # Run tests that use live data or depend on external systems + # This should not be mandatory as part of the PR process to not have + # a blocker because said external system is down. + build_and_test_live_data: + strategy: + matrix: + node: [16] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v2.3.3 + + - name: Install NodeJS + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + + - name: Cache npm cache + uses: actions/cache@v2 + with: + path: ~/.npm + key: node-${{ matrix.os }}-${{ matrix.node }}-v1-${{ hashFiles('**/package-lock.json') }} + + - name: install using npm ci + uses: bahmutov/npm-install@v1 + + - name: karma live data tests + env: + DEBUG: "waku:test*" + run: npm run test:karma-live-data + + - name: Upload logs on failure + uses: actions/upload-artifact@v2 + if: failure() + with: + name: nim-waku-logs + path: log/ diff --git a/karma-live-data.conf.js b/karma-live-data.conf.js new file mode 100644 index 0000000000..cb420e48fc --- /dev/null +++ b/karma-live-data.conf.js @@ -0,0 +1,45 @@ +process.env.CHROME_BIN = require('puppeteer').executablePath(); + +module.exports = function (config) { + config.set({ + frameworks: ['mocha', 'karma-typescript'], + files: ['src/lib/**/*.ts', 'src/proto/**/*.ts'], + preprocessors: { + '**/*.ts': ['karma-typescript'], + }, + plugins: [ + require('karma-mocha'), + require('karma-typescript'), + require('karma-chrome-launcher'), + ], + reporters: ['progress', 'karma-typescript'], + browsers: ['ChromeHeadless'], + singleRun: true, + client: { + mocha: { + timeout: 6000, // Default is 2s + }, + args: ['--grep', '[live data]]'], + }, + karmaTypescriptConfig: { + bundlerOptions: { + entrypoints: /^.*[^(node)]\.spec\.ts$/, + }, + coverageOptions: { + instrumentation: false, + }, + tsconfig: './tsconfig.json', + compilerOptions: { + noEmit: false, + }, + include: { + mode: 'replace', + values: ['src/lib/**/*.ts', 'src/proto/**/*.ts'], + }, + exclude: { + mode: 'replace', + values: ['node_modules/**'], + }, + }, + }); +}; diff --git a/package.json b/package.json index 9e9495390e..3480f3333c 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "test:spelling": "cspell \"{README.md,.github/*.md,guides/*.md,src/**/*.ts}\"", "test:unit": "nyc --silent mocha", "test:karma": "karma start", + "test:karma-live-data": "LIVE_DATA_TESTS=true karma start ./karma-live-data.conf.js", "examples:test": "run-s examples:pretest; for d in examples/*/; do (cd $d && npm test;); done", "proto": "run-s proto:*", "proto:lint": "buf lint", diff --git a/src/lib/discovery/dns.spec.ts b/src/lib/discovery/dns.spec.ts index 2563c70ade..964b36d457 100644 --- a/src/lib/discovery/dns.spec.ts +++ b/src/lib/discovery/dns.spec.ts @@ -176,6 +176,12 @@ describe('DNS Node Discovery [live data]', function () { const ipTestRegex = /^\d+\.\d+\.\d+\.\d+$/; const maxQuantity = 3; + before(function () { + if (process.env.CI && !process.env.LIVE_DATA_TESTS) { + this.skip(); + } + }); + it(`should retrieve ${maxQuantity} PeerInfos for test.nodes.vac.dev`, async function () { this.timeout(5000); // Google's dns server address. Needs to be set explicitly to run in CI From 8e591f22db9d8479332f163cf8c048a082e73730 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Fri, 14 Jan 2022 12:49:40 +1100 Subject: [PATCH 15/16] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7f5f6a86..bea78c2803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Implement DNS Discovery as per [EIP-1459](https://eips.ethereum.org/EIPS/eip-1459), + with ENR records as defined in [31/WAKU2-ENR](https://rfc.vac.dev/spec/31/); + Available by passing `{ bootstrap: { enrUrl: enrtree://... } }` to `Waku.create`. + ### Changed - Test: Upgrade nim-waku node to v0.6. From 99763322db6644f883d18dfbee87315ff0fc108d Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Fri, 14 Jan 2022 12:57:29 +1100 Subject: [PATCH 16/16] Import Karma settings instead of duplicating them --- karma-live-data.conf.js | 55 ++++++++++------------------------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/karma-live-data.conf.js b/karma-live-data.conf.js index cb420e48fc..4d2d777bac 100644 --- a/karma-live-data.conf.js +++ b/karma-live-data.conf.js @@ -1,45 +1,16 @@ -process.env.CHROME_BIN = require('puppeteer').executablePath(); +// import settings from default config file +let properties = null; +const originalConfigFn = require('./karma.conf.js'); +originalConfigFn({ + set: function (arg) { + properties = arg; + }, +}); +// pass `--grep '[live data]'` to mocha to only run live data tests +properties.client.args = ['--grep', '[live data]]']; + +// export settings module.exports = function (config) { - config.set({ - frameworks: ['mocha', 'karma-typescript'], - files: ['src/lib/**/*.ts', 'src/proto/**/*.ts'], - preprocessors: { - '**/*.ts': ['karma-typescript'], - }, - plugins: [ - require('karma-mocha'), - require('karma-typescript'), - require('karma-chrome-launcher'), - ], - reporters: ['progress', 'karma-typescript'], - browsers: ['ChromeHeadless'], - singleRun: true, - client: { - mocha: { - timeout: 6000, // Default is 2s - }, - args: ['--grep', '[live data]]'], - }, - karmaTypescriptConfig: { - bundlerOptions: { - entrypoints: /^.*[^(node)]\.spec\.ts$/, - }, - coverageOptions: { - instrumentation: false, - }, - tsconfig: './tsconfig.json', - compilerOptions: { - noEmit: false, - }, - include: { - mode: 'replace', - values: ['src/lib/**/*.ts', 'src/proto/**/*.ts'], - }, - exclude: { - mode: 'replace', - values: ['node_modules/**'], - }, - }, - }); + config.set(properties); };