diff --git a/.cspell.json b/.cspell.json index ab5c739a25..3a3dd3ab16 100644 --- a/.cspell.json +++ b/.cspell.json @@ -23,8 +23,9 @@ "Dscore", "ecies", "editorconfig", - "ENR", - "ENRs", + "enr", + "enrs", + "enrtree", "ephem", "esnext", "ethersproject", @@ -61,6 +62,7 @@ "muxer", "mvps", "nodekey", + "opendns", "peerhave", "prettierignore", "proto", @@ -71,9 +73,11 @@ "rlnrelay", "roadmap", "sandboxed", + "scanf", "secio", "seckey", "secp", + "sscanf", "staticnode", "statusim", "submodule", diff --git a/package-lock.json b/package-lock.json index 8d266c697a..aa3d8afbe0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", @@ -2106,6 +2108,11 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "node_modules/@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -3118,6 +3125,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "node_modules/@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -6183,6 +6198,42 @@ "receptacle": "^1.3.2" } }, + "node_modules/dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + }, + "bin": { + "dns-query": "bin/dns-query" + } + }, + "node_modules/dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "dependencies": { + "dns-packet": "^5.2.4" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -8368,6 +8419,11 @@ "node": ">= 8" } }, + "node_modules/hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -19349,6 +19405,11 @@ "chalk": "^4.0.0" } }, + "@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -20233,6 +20294,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "requires": { + "@types/node": "*" + } + }, "@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -22697,6 +22766,33 @@ "receptacle": "^1.3.2" } }, + "dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.1" + } + }, + "dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + } + }, + "dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "requires": { + "dns-packet": "^5.2.4" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -24357,6 +24453,11 @@ } } }, + "hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", diff --git a/package.json b/package.json index 7202a433f4..5686982d5d 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts new file mode 100644 index 0000000000..6ba0860bc0 --- /dev/null +++ b/src/lib/discovery/bootstrap.ts @@ -0,0 +1,115 @@ +import debug from 'debug'; + +import { DnsNodeDiscovery } from './dns'; + +import { getNodesFromHostedJson, getPseudoRandomSubset } from './index'; + +const dbg = debug('waku:discovery:bootstrap'); + +const DefaultMaxPeers = 1; + +/** + * Setup discovery method used to bootstrap. + * + * Only one method is used. `default`, `peers`, `getPeers` and `enr` options are mutually exclusive. + */ +export interface BootstrapOptions { + /** + * The maximum of peers to connect to as part of the bootstrap process. + * + * @default 1 + */ + maxPeers?: number; + /** + * Use the default discovery method. + * + * The default discovery method is likely to change overtime as new discovery + * methods are implemented. + * + * @default false + */ + default?: boolean; + /** + * Multiaddrs of peers to connect to. + */ + peers?: string[]; + /** + * Getter that retrieve multiaddrs of peers to connect to. + */ + getPeers?: () => Promise; + /** + * An EIP-1459 ENR Tree URL. For example: + * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev" + */ + enrUrl?: string; +} + +/** + * Parse the bootstrap options and returns an async function that returns node addresses upon invocation. + */ +export function parseBootstrap( + options: + | BootstrapOptions + | boolean + | string[] + | (() => string[] | Promise) +): undefined | (() => Promise) { + if ( + Object.keys(options).includes('default') || + Object.keys(options).includes('maxPeers') || + Object.keys(options).includes('peers') || + Object.keys(options).includes('getPeers') || + Object.keys(options).includes('enrUrl') + ) { + const opts = options as unknown as BootstrapOptions; + const maxPeers = opts.maxPeers || DefaultMaxPeers; + + if (opts.default) { + dbg('Bootstrap: Use hosted list of peers.'); + + return getNodesFromHostedJson.bind({}, undefined, undefined, maxPeers); + } else if (opts.peers !== undefined && opts.peers.length > 0) { + dbg('Bootstrap: Use provided list of peers.'); + + const allPeers: string[] = opts.peers; + return (): Promise => { + const peers = getPseudoRandomSubset(allPeers, maxPeers); + return Promise.resolve(peers); + }; + } else if (typeof opts.getPeers === 'function') { + dbg('Bootstrap: Use provided getPeers function.'); + const getPeers = opts.getPeers; + + return async (): Promise => { + const allPeers = await getPeers(); + return getPseudoRandomSubset(allPeers, maxPeers); + }; + } else if (opts.enrUrl) { + const enrUrl = opts.enrUrl; + dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); + + const dns = DnsNodeDiscovery.dnsOverHttp(); + // TODO: The `toString` is incorrect. + return (): Promise => + dns + .getPeers(maxPeers, [enrUrl]) + .then((peers) => peers.map((peer) => peer.toString())); + } + } else { + dbg( + 'WARN: This bootstrap method will be deprecated, use `BootstrapOptions` instead' + ); + if (options === true) { + return getNodesFromHostedJson; + } else if (Array.isArray(options)) { + return (): Promise => { + return Promise.resolve(options as string[]); + }; + } else if (typeof options === 'function') { + return async (): Promise => { + return options(); + }; + } + } + return; +} diff --git a/src/lib/discovery/dns.spec.ts b/src/lib/discovery/dns.spec.ts new file mode 100644 index 0000000000..2563c70ade --- /dev/null +++ b/src/lib/discovery/dns.spec.ts @@ -0,0 +1,197 @@ +import { expect } from 'chai'; + +import { DnsClient, DnsNodeDiscovery } from './dns'; +import testData from './testdata.json'; + +const mockData = testData.dns; + +const host = 'nodes.example.org'; +const rootDomain = 'JORXBYVVM7AEKETX5DGXW44EAY'; +const branchDomainA = 'D2SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainB = 'D3SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainC = 'D4SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainD = 'D5SNLTAGWNQ34NTQTPHNZDECFU'; +const partialBranchA = 'AAAA'; +const partialBranchB = 'BBBB'; +const singleBranch = `enrtree-branch:${branchDomainA}`; +const doubleBranch = `enrtree-branch:${branchDomainA},${branchDomainB}`; +const multiComponentBranch = [ + `enrtree-branch:${branchDomainA},${partialBranchA}`, + `${partialBranchB},${branchDomainB}`, +]; + +// Note: once td.when is asked to throw for an input it will always throw. +// Input can't be re-used for a passing case. +const errorBranchA = `enrtree-branch:${branchDomainC}`; +const errorBranchB = `enrtree-branch:${branchDomainD}`; + +/** + * Mocks DNS resolution. + */ +class MockDNS implements DnsClient { + fqdnRes: Map; + fqdnThrows: string[]; + + constructor() { + this.fqdnRes = new Map(); + this.fqdnThrows = []; + } + + addRes(fqdn: string, res: string[]): void { + this.fqdnRes.set(fqdn, res); + } + + addThrow(fqdn: string): void { + this.fqdnThrows.push(fqdn); + } + + resolveTXT(fqdn: string): Promise { + if (this.fqdnThrows.includes(fqdn)) throw 'Mock DNS throws.'; + + const res = this.fqdnRes.get(fqdn); + + if (!res) throw `Mock DNS could not resolve ${fqdn}`; + + return Promise.resolve(res); + } +} + +describe('DNS Node Discovery', () => { + let mockDns: MockDNS; + + beforeEach(() => { + mockDns = new MockDNS(); + mockDns.addRes(host, [mockData.enrRoot]); + }); + + it('retrieves a single peer', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(1); + expect(peers[0].ip).to.eq('45.77.40.127'); + expect(peers[0].tcp).to.eq(30303); + }); + + it('retrieves all peers (2) when maxQuantity larger than DNS tree size', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [doubleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrB]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(2); + expect(peers[0].ip).to.not.eq(peers[1].ip); + }); + + it('retrieves all peers (3) when branch entries are composed of multiple strings', async function () { + mockDns.addRes(`${rootDomain}.${host}`, multiComponentBranch); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enr]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrA]); + mockDns.addRes(`${partialBranchA}${partialBranchB}.${host}`, [ + mockData.enrB, + ]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(3); + expect(peers[0].ip).to.not.eq(peers[1].ip); + expect(peers[0].ip).to.not.eq(peers[2].ip); + expect(peers[1].ip).to.not.eq(peers[2].ip); + }); + + it('it tolerates circular branch references', async function () { + // root --> branchA + // branchA --> branchA + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [singleBranch]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + }); + + it('recovers when dns.resolve returns empty', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + + // Empty response case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + let peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + + // No TXT records case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores domain fetching errors', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchA]); + mockDns.addThrow(`${branchDomainC}.${host}`); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores unrecognized TXT record formats', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [mockData.enrBranchBadPrefix]); + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('caches peers it previously fetched', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchB]); + mockDns.addRes(`${branchDomainD}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peersA = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersA.length).to.eq(1); + + // Specify that a subsequent network call retrieving the same peer should throw. + // This test passes only if the peer is fetched from cache + mockDns.addThrow(`${branchDomainD}.${host}`); + + const peersB = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersB.length).to.eq(1); + expect(peersA[0].ip).to.eq(peersB[0].ip); + }); +}); + +describe('DNS Node Discovery [live data]', function () { + const publicKey = 'AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C'; + const fqdn = 'test.nodes.vac.dev'; + const enrTree = `enrtree://${publicKey}@${fqdn}`; + const ipTestRegex = /^\d+\.\d+\.\d+\.\d+$/; + const maxQuantity = 3; + + it(`should retrieve ${maxQuantity} PeerInfos for test.nodes.vac.dev`, async function () { + this.timeout(5000); + // Google's dns server address. Needs to be set explicitly to run in CI + const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); + const peers = await dnsNodeDiscovery.getPeers(maxQuantity, [enrTree]); + + expect(peers.length).to.eq(maxQuantity); + + // TODO: Test multiaddrs entry + console.log(peers.map((peer) => peer.multiaddrs)); + + const seen: string[] = []; + for (const peer of peers) { + expect(peer!.ip!).to.match(ipTestRegex); + expect(seen).to.not.include(peer!.ip!); + seen.push(peer!.ip!); + } + }); +}); diff --git a/src/lib/discovery/dns.ts b/src/lib/discovery/dns.ts new file mode 100644 index 0000000000..ba3efd00f6 --- /dev/null +++ b/src/lib/discovery/dns.ts @@ -0,0 +1,201 @@ +import assert from 'assert'; + +import { debug } from 'debug'; + +import { ENR } from '../enr'; + +import { DnsOverHttps, Endpoints } from './dns_over_https'; +import { ENRTree } from './enrtree'; + +const dbg = debug('waku:discovery:dns'); + +type SearchContext = { + domain: string; + publicKey: string; + visits: { [key: string]: boolean }; +}; + +export interface DnsClient { + resolveTXT: (domain: string) => Promise; +} + +export class DnsNodeDiscovery { + private readonly dns: DnsClient; + private readonly _DNSTreeCache: { [key: string]: string }; + private readonly _errorTolerance: number = 10; + + public static dnsOverHttp(endpoints?: Endpoints): DnsNodeDiscovery { + const dnsClient = new DnsOverHttps(endpoints); + return new DnsNodeDiscovery(dnsClient); + } + + /** + * Returns a list of verified peers listed in an EIP-1459 DNS tree. Method may + * return fewer peers than requested if `maxQuantity` is larger than the number + * of ENR records or the number of errors/duplicate peers encountered by randomized + * search exceeds `maxQuantity` plus the `errorTolerance` factor. + */ + async getPeers(maxQuantity: number, enrTreeUrls: string[]): Promise { + let totalSearches = 0; + const peers: ENR[] = []; + + const networkIndex = Math.floor(Math.random() * enrTreeUrls.length); + const { publicKey, domain } = ENRTree.parseTree(enrTreeUrls[networkIndex]); + + while ( + peers.length < maxQuantity && + totalSearches < maxQuantity + this._errorTolerance + ) { + const context: SearchContext = { + domain, + publicKey, + visits: {}, + }; + + const peer = await this._search(domain, context); + + if (peer && isNewPeer(peer, peers)) { + peers.push(peer); + dbg( + `got new peer candidate from DNS address=${peer.nodeId}@${peer.ip}` + ); + } + + totalSearches++; + } + return peers; + } + + public constructor(dns: DnsClient) { + this._DNSTreeCache = {}; + this.dns = dns; + } + + /** + * Runs a recursive, randomized descent of the DNS tree to retrieve a single + * ENR record as an ENR. Returns null if parsing or DNS resolution fails. + */ + private async _search( + subdomain: string, + context: SearchContext + ): Promise { + const entry = await this._getTXTRecord(subdomain, context); + context.visits[subdomain] = true; + + let next: string; + let branches: string[]; + + const entryType = getEntryType(entry); + try { + switch (entryType) { + case ENRTree.ROOT_PREFIX: + next = ENRTree.parseAndVerifyRoot(entry, context.publicKey); + return await this._search(next, context); + case ENRTree.BRANCH_PREFIX: + branches = ENRTree.parseBranch(entry); + next = selectRandomPath(branches, context); + return await this._search(next, context); + case ENRTree.RECORD_PREFIX: + return ENR.decodeTxt(entry); + default: + return null; + } + } catch (error) { + dbg( + `Failed to search DNS tree ${entryType} at subdomain ${subdomain}: ${error}` + ); + return null; + } + } + + /** + * Retrieves the TXT record stored at a location from either + * this DNS tree cache or via DNS query + */ + private async _getTXTRecord( + subdomain: string, + context: SearchContext + ): Promise { + if (this._DNSTreeCache[subdomain]) { + return this._DNSTreeCache[subdomain]; + } + + // Location is either the top level tree entry host or a subdomain of it. + const location = + subdomain !== context.domain + ? `${subdomain}.${context.domain}` + : context.domain; + + const response = await this.dns.resolveTXT(location); + + assert( + response.length, + 'Received empty result array while fetching TXT record' + ); + assert(response[0].length, 'Received empty TXT record'); + + // Branch entries can be an array of strings of comma delimited subdomains, with + // some subdomain strings split across the array elements + const result = response.join(''); + + this._DNSTreeCache[subdomain] = result; + return result; + } +} + +function getEntryType(entry: string): string { + if (entry.startsWith(ENRTree.ROOT_PREFIX)) return ENRTree.ROOT_PREFIX; + if (entry.startsWith(ENRTree.BRANCH_PREFIX)) return ENRTree.BRANCH_PREFIX; + if (entry.startsWith(ENRTree.RECORD_PREFIX)) return ENRTree.RECORD_PREFIX; + + return ''; +} + +/** + * Returns a randomly selected subdomain string from the list provided by a branch + * entry record. + * + * The client must track subdomains which are already resolved to avoid + * going into an infinite loop b/c branch entries can contain + * circular references. It’s in the client’s best interest to traverse the + * tree in random order. + */ +function selectRandomPath(branches: string[], context: SearchContext): string { + // Identify domains already visited in this traversal of the DNS tree. + // Then filter against them to prevent cycles. + const circularRefs: { [key: number]: boolean } = {}; + for (const [idx, subdomain] of branches.entries()) { + if (context.visits[subdomain]) { + circularRefs[idx] = true; + } + } + // If all possible paths are circular... + if (Object.keys(circularRefs).length === branches.length) { + throw new Error('Unresolvable circular path detected'); + } + + // Randomly select a viable path + let index; + do { + index = Math.floor(Math.random() * branches.length); + } while (circularRefs[index]); + + return branches[index]; +} + +/** + * @returns false if candidate peer already exists in the + * current collection of peers based on the node id value; + * true otherwise. + */ +function isNewPeer(peer: ENR | null, peers: ENR[]): boolean { + if (!peer || !peer.nodeId) return false; + + for (const existingPeer of peers) { + if (peer.nodeId === existingPeer.nodeId) { + return false; + } + } + + return true; +} diff --git a/src/lib/discovery/dns_over_https.ts b/src/lib/discovery/dns_over_https.ts new file mode 100644 index 0000000000..d71fb80130 --- /dev/null +++ b/src/lib/discovery/dns_over_https.ts @@ -0,0 +1,60 @@ +import { TxtAnswer } from 'dns-packet'; +import { + endpoints as defaultEndpoints, + Endpoint, + EndpointProps, + query, +} from 'dns-query'; + +import { DnsClient } from './dns'; + +const { cloudflare, google, opendns } = defaultEndpoints; + +export type Endpoints = + | 'doh' + | 'dns' + | Iterable; + +export class DnsOverHttps implements DnsClient { + /** + * Create new Dns-Over-Http DNS client. + * + * @param endpoints The endpoints for Dns-Over-Https queries. + * See [dns-query](https://www.npmjs.com/package/dns-query) for details. + * + * @throws {code: string} If DNS query fails. + */ + public constructor( + public endpoints: Endpoints = [cloudflare, google, opendns] + ) {} + + async resolveTXT(domain: string): Promise { + const response = await query({ + questions: [{ type: 'TXT', name: domain }], + }); + + const answers = response.answers as TxtAnswer[]; + + const data = answers.map((a) => a.data); + + const result: string[] = []; + + data.forEach((d) => { + if (typeof d === 'string') { + result.push(d); + } else if (Array.isArray(d)) { + d.forEach((sd) => { + if (typeof sd === 'string') { + result.push(sd); + } else { + result.push(Buffer.from(sd).toString('utf-8')); + } + }); + } else { + result.push(Buffer.from(d).toString('utf-8')); + } + }); + + return result; + } +} diff --git a/src/lib/discovery/enrtree.spec.ts b/src/lib/discovery/enrtree.spec.ts new file mode 100644 index 0000000000..af7402b82e --- /dev/null +++ b/src/lib/discovery/enrtree.spec.ts @@ -0,0 +1,89 @@ +import { expect } from 'chai'; + +import { ENRTree } from './enrtree'; +import testData from './testdata.json'; + +const dns = testData.dns; + +describe('ENRTree', () => { + // Root DNS entries + it('ENRTree (root): should parse and verify and DNS root entry', () => { + const subdomain = ENRTree.parseAndVerifyRoot(dns.enrRoot, dns.publicKey); + + expect(subdomain).to.eq('JORXBYVVM7AEKETX5DGXW44EAY'); + }); + + it('ENRTree (root): should error if DNS root entry is mis-prefixed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadPrefix, dns.publicKey); + } catch (e) { + expect(e.toString()).includes( + "ENRTree root entry must start with 'enrtree-root:'" + ); + } + }); + + it('ENRTree (root): should error if DNS root entry signature is invalid', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadSig, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Unable to verify ENRTree root signature'); + } + }); + + it('ENRTree (root): should error if DNS root entry is malformed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootMalformed, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree root entry'); + } + }); + + // Tree DNS entries + it('ENRTree (tree): should parse a DNS tree entry', () => { + const { publicKey, domain } = ENRTree.parseTree(dns.enrTree); + + expect(publicKey).to.eq(dns.publicKey); + expect(domain).to.eq('nodes.example.org'); + }); + + it('ENRTree (tree): should error if DNS tree entry is mis-prefixed', () => { + try { + ENRTree.parseTree(dns.enrTreeBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree tree entry must start with 'enrtree:'" + ); + } + }); + + it('ENRTree (tree): should error if DNS tree entry is misformatted', () => { + try { + ENRTree.parseTree(dns.enrTreeMalformed); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree tree entry'); + } + }); + + // Branch entries + it('ENRTree (branch): should parse and verify a single component DNS branch entry', () => { + const expected = [ + 'D2SNLTAGWNQ34NTQTPHNZDECFU', + '67BLTJEU5R2D5S3B4QKJSBRFCY', + 'A2HDMZBB4JIU53VTEGC4TG6P4A', + ]; + + const branches = ENRTree.parseBranch(dns.enrBranch); + expect(branches).to.deep.eq(expected); + }); + + it('ENRTree (branch): should error if DNS branch entry is mis-prefixed', () => { + try { + ENRTree.parseBranch(dns.enrBranchBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree branch entry must start with 'enrtree-branch:'" + ); + } + }); +}); diff --git a/src/lib/discovery/enrtree.ts b/src/lib/discovery/enrtree.ts new file mode 100644 index 0000000000..d256cab9ec --- /dev/null +++ b/src/lib/discovery/enrtree.ts @@ -0,0 +1,123 @@ +import assert from 'assert'; + +import base64url from 'base64url'; +import * as base32 from 'hi-base32'; +import { ecdsaVerify } from 'secp256k1'; + +import { ENR } from '../enr'; +import { keccak256Buf } from '../utils'; + +export interface PeerInfo { + id?: Uint8Array | Buffer; + address?: string; + udpPort?: number | null; + tcpPort?: number | null; +} + +type ENRRootValues = { + eRoot: string; + lRoot: string; + seq: number; + signature: string; +}; + +type ENRTreeValues = { + publicKey: string; + domain: string; +}; + +export class ENRTree { + public static readonly RECORD_PREFIX = ENR.RECORD_PREFIX; + public static readonly TREE_PREFIX = 'enrtree:'; + public static readonly BRANCH_PREFIX = 'enrtree-branch:'; + public static readonly ROOT_PREFIX = 'enrtree-root:'; + + /** + * Extracts the branch subdomain referenced by a DNS tree root string after verifying + * the root record signature with its base32 compressed public key. + */ + static parseAndVerifyRoot(root: string, publicKey: string): string { + assert( + root.startsWith(this.ROOT_PREFIX), + `ENRTree root entry must start with '${this.ROOT_PREFIX}'` + ); + + const rootValues = ENRTree.parseRootValues(root); + const decodedPublicKey = base32.decode.asBytes(publicKey); + + // The signature is a 65-byte secp256k1 over the keccak256 hash + // of the record content, excluding the `sig=` part, encoded as URL-safe base64 string + // (Trailing recovery bit must be trimmed to pass `ecdsaVerify` method) + const signedComponent = root.split(' sig')[0]; + const signedComponentBuffer = Buffer.from(signedComponent); + const signatureBuffer = base64url + .toBuffer(rootValues.signature) + .slice(0, 64); + const keyBuffer = Buffer.from(decodedPublicKey); + + const isVerified = ecdsaVerify( + signatureBuffer, + keccak256Buf(signedComponentBuffer), + keyBuffer + ); + + assert(isVerified, 'Unable to verify ENRTree root signature'); + + return rootValues.eRoot; + } + + static parseRootValues(txt: string): ENRRootValues { + const matches = txt.match( + /^enrtree-root:v1 e=([^ ]+) l=([^ ]+) seq=(\d+) sig=([^ ]+)$/ + ); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree root entry'); + + matches.shift(); // The first entry is the full match + const [eRoot, lRoot, seq, signature] = matches; + + assert.ok(eRoot, "Could not parse 'e' value from ENRTree root entry"); + assert.ok(lRoot, "Could not parse 'l' value from ENRTree root entry"); + assert.ok(seq, "Could not parse 'seq' value from ENRTree root entry"); + assert.ok(signature, "Could not parse 'sig' value from ENRTree root entry"); + + return { eRoot, lRoot, seq: Number(seq), signature }; + } + + /** + * Returns the public key and top level domain of an ENR tree entry. + * The domain is the starting point for traversing a set of linked DNS TXT records + * and the public key is used to verify the root entry record + */ + static parseTree(tree: string): ENRTreeValues { + assert( + tree.startsWith(this.TREE_PREFIX), + `ENRTree tree entry must start with '${this.TREE_PREFIX}'` + ); + + const matches = tree.match(/^enrtree:\/\/([^@]+)@(.+)$/); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree tree entry'); + + matches.shift(); // The first entry is the full match + const [publicKey, domain] = matches; + + assert.ok(publicKey, 'Could not parse public key from ENRTree tree entry'); + assert.ok(domain, 'Could not parse domain from ENRTree tree entry'); + + return { publicKey, domain }; + } + + /** + * Returns subdomains listed in an ENR branch entry. These in turn lead to + * either further branch entries or ENR records. + */ + static parseBranch(branch: string): string[] { + assert( + branch.startsWith(this.BRANCH_PREFIX), + `ENRTree branch entry must start with '${this.BRANCH_PREFIX}'` + ); + + return branch.split(this.BRANCH_PREFIX)[1].split(','); + } +} diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts index 09eeb32682..5513e332c0 100644 --- a/src/lib/discovery/index.ts +++ b/src/lib/discovery/index.ts @@ -1,6 +1,7 @@ import { shuffle } from 'libp2p-gossipsub/src/utils'; export { getNodesFromHostedJson } from './hosted_json'; +export { parseBootstrap } from './bootstrap'; export function getPseudoRandomSubset( values: string[], diff --git a/src/lib/discovery/testdata.json b/src/lib/discovery/testdata.json new file mode 100644 index 0000000000..86a3a7b1be --- /dev/null +++ b/src/lib/discovery/testdata.json @@ -0,0 +1,18 @@ +{ + "dns": { + "publicKey": "AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE", + "enr": "enr:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrA": "enr:-Jq4QAopXcF_SSfOwl_AmLdrMUnHQO1Rx-XV4gYeySSK32PTbQ8volkh3IQy1ag1Gkl6O-C5rjskj3EyDi8XVzck4PMVg2V0aMrJhKALwySDbxWAgmlkgnY0gmlwhC1NKH-Jc2VjcDI1NmsxoQO5wMEjJLtqT-h6zhef0xsO-SW-pcQD-yuNqCr3GTEZFoN0Y3CCdl-DdWRwgnZf", + "enrB": "enr:-Je4QAFx_6rFjCxCLPUbxIA_KS7FhCYeTU6fXmbj1V08f8DPCUAB9bLoY2Yy7q2hIEby7Yf6e_v7gbofloB1oTnjqeYDg2V0aMfGhOAp6ZGAgmlkgnY0gmlwhLxf-D2Jc2VjcDI1NmsxoQOou7vgUXL96E5CzBsCE6N1GSMqlAACtUxRiNpq6vnB6IN0Y3CCdl-DdWRwgnZf", + "enrRoot": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranch": "enrtree-branch:D2SNLTAGWNQ34NTQTPHNZDECFU,67BLTJEU5R2D5S3B4QKJSBRFCY,A2HDMZBB4JIU53VTEGC4TG6P4A", + "enrTree": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@nodes.example.org", + "enrBadPrefix": "enrabc:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrRootBadPrefix": "enrtree:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranchBadPrefix": "Z64M,JOECK7UUYUFVX24QGXYLR3UHDU,RR6SC4GUZBKLFA2WO4IUY6YGEE,EQRME5EAOS7AJHHLDDZNDYT7GI,JXHUMLDSGKU6UQWYFMNCFYQFHQ,4SNDLPNM3CBG2KLBMRSTHWFNP4,WEEEFCKUXOGU4QPKCRBBEHQLEY,CPXM5AOSTICZ3TODJFQACGBWMU,7U26GD37NS6DV72PDAURZI4WUY,MYLQIGMR5GTKPPBMXIINZ2ALGU", + "enrTreeBadPrefix": "entree-branch://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@nodes.example.org", + "enrRootBadSig": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Aa7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrRootMalformed": "enrtree-root:v1 e=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrTreeMalformed": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2nodes.example.org" + } +} diff --git a/src/lib/enr/enr.spec.ts b/src/lib/enr/enr.spec.ts index e9e4d133c5..c7feb48610 100644 --- a/src/lib/enr/enr.spec.ts +++ b/src/lib/enr/enr.spec.ts @@ -113,7 +113,7 @@ describe('ENR', function () { ENR.decodeTxt(txt); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); }); @@ -153,7 +153,7 @@ describe('ENR', function () { enr.verify(Buffer.alloc(0), Buffer.alloc(0)); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); diff --git a/src/lib/enr/enr.ts b/src/lib/enr/enr.ts index 440ca16ba2..d41e6584b2 100644 --- a/src/lib/enr/enr.ts +++ b/src/lib/enr/enr.ts @@ -25,6 +25,7 @@ import { ENRKey, ENRValue, NodeId, SequenceNumber } from './types'; import * as v4 from './v4'; export class ENR extends Map { + public static readonly RECORD_PREFIX = 'enr:'; public seq: SequenceNumber; public signature: Buffer | null; @@ -93,8 +94,10 @@ export class ENR extends Map { } static decodeTxt(encoded: string): ENR { - if (!encoded.startsWith('enr:')) { - throw new Error("string encoded ENR must start with 'enr:'"); + if (!encoded.startsWith(this.RECORD_PREFIX)) { + throw new Error( + `"string encoded ENR must start with '${this.RECORD_PREFIX}'` + ); } return ENR.decode(base64url.toBuffer(encoded.slice(4))); } @@ -388,6 +391,7 @@ export class ENR extends Map { return new Multiaddr(maBuf); } + setLocationMultiaddr(multiaddr: Multiaddr): void { const protoNames = multiaddr.protoNames(); if ( @@ -428,7 +432,7 @@ export class ENR extends Map { throw new Error(ERR_INVALID_ID); } if (!this.publicKey) { - throw new Error('Failed to verify enr: No public key'); + throw new Error('Failed to verify ENR: No public key'); } return v4.verify(this.publicKey, data, signature); } @@ -471,6 +475,6 @@ export class ENR extends Map { } encodeTxt(privateKey?: Buffer): string { - return 'enr:' + base64url.encode(this.encode(privateKey)); + return ENR.RECORD_PREFIX + base64url.encode(this.encode(privateKey)); } } diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 605e5b95dc..398e450c31 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,3 +1,5 @@ +import { keccak256, Message } from 'js-sha3'; + export function hexToBuf(hex: string | Buffer | Uint8Array): Buffer { if (typeof hex === 'string') { return Buffer.from(hex.replace(/^0x/i, ''), 'hex'); @@ -31,3 +33,7 @@ export function equalByteArrays( return aBuf.compare(bBuf) === 0; } + +export function keccak256Buf(message: Message): Buffer { + return Buffer.from(keccak256.arrayBuffer(message)); +} diff --git a/src/lib/waku.ts b/src/lib/waku.ts index f5da1c5d3e..de942c89e7 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -18,7 +18,7 @@ import Ping from 'libp2p/src/ping'; import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; -import { getNodesFromHostedJson } from './discovery'; +import { parseBootstrap } from './discovery'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; import { WakuMessage } from './waku_message'; @@ -86,10 +86,7 @@ export interface CreateOptions { /** * Use libp2p-bootstrap to discover and connect to new nodes. * - * You can pass: - * - `true` to use {@link getNodesFromHostedJson}, - * - an array of multiaddresses, - * - a function that returns an array of multiaddresses (or Promise of). + * See [BootstrapOptions] for available parameters. * * Note: It overrides any other peerDiscovery modules that may have been set via * {@link CreateOptions.libp2p}. @@ -189,17 +186,7 @@ export class Waku { }); if (options?.bootstrap) { - let bootstrap: undefined | (() => string[] | Promise); - - if (options.bootstrap === true) { - bootstrap = getNodesFromHostedJson; - } else if (Array.isArray(options.bootstrap)) { - bootstrap = (): string[] => { - return options.bootstrap as string[]; - }; - } else if (typeof options.bootstrap === 'function') { - bootstrap = options.bootstrap; - } + const bootstrap = parseBootstrap(options?.bootstrap); if (bootstrap !== undefined) { try {