diff --git a/.cspell.json b/.cspell.json index ab5c739a25..3a3dd3ab16 100644 --- a/.cspell.json +++ b/.cspell.json @@ -23,8 +23,9 @@ "Dscore", "ecies", "editorconfig", - "ENR", - "ENRs", + "enr", + "enrs", + "enrtree", "ephem", "esnext", "ethersproject", @@ -61,6 +62,7 @@ "muxer", "mvps", "nodekey", + "opendns", "peerhave", "prettierignore", "proto", @@ -71,9 +73,11 @@ "rlnrelay", "roadmap", "sandboxed", + "scanf", "secio", "seckey", "secp", + "sscanf", "staticnode", "statusim", "submodule", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b6d8049f1..11ab36c6e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,3 +90,42 @@ jobs: with: name: nim-waku-logs path: log/ + + # Run tests that use live data or depend on external systems + # This should not be mandatory as part of the PR process to not have + # a blocker because said external system is down. + build_and_test_live_data: + strategy: + matrix: + node: [16] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v2.3.3 + + - name: Install NodeJS + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + + - name: Cache npm cache + uses: actions/cache@v2 + with: + path: ~/.npm + key: node-${{ matrix.os }}-${{ matrix.node }}-v1-${{ hashFiles('**/package-lock.json') }} + + - name: install using npm ci + uses: bahmutov/npm-install@v1 + + - name: karma live data tests + env: + DEBUG: "waku:test*" + run: npm run test:karma-live-data + + - name: Upload logs on failure + uses: actions/upload-artifact@v2 + if: failure() + with: + name: nim-waku-logs + path: log/ diff --git a/CHANGELOG.md b/CHANGELOG.md index af97df1838..bea78c2803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Implement DNS Discovery as per [EIP-1459](https://eips.ethereum.org/EIPS/eip-1459), + with ENR records as defined in [31/WAKU2-ENR](https://rfc.vac.dev/spec/31/); + Available by passing `{ bootstrap: { enrUrl: enrtree://... } }` to `Waku.create`. + ### Changed - Test: Upgrade nim-waku node to v0.6. +- **Breaking**: Renamed `getBootstrapNodes` to `getNodesFromHostedJson`. - Minimum node version changed to 16. +- **Breaking**: Changed `Waku.create` bootstrap option from `{ bootstrap: boolean }` to `{ bootstrap: BootstrapOptions }`. + Replace `{ boostrap: true }` with `{ boostrap: { default: true } }` to retain same behaviour. ### Fixed diff --git a/examples/eth-pm-wallet-encryption/package-lock.json b/examples/eth-pm-wallet-encryption/package-lock.json index aa98ab43c8..9fab46b77e 100644 --- a/examples/eth-pm-wallet-encryption/package-lock.json +++ b/examples/eth-pm-wallet-encryption/package-lock.json @@ -6440,9 +6440,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27967,9 +27967,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", diff --git a/examples/eth-pm-wallet-encryption/src/waku.ts b/examples/eth-pm-wallet-encryption/src/waku.ts index d0be5baf83..f808dff396 100644 --- a/examples/eth-pm-wallet-encryption/src/waku.ts +++ b/examples/eth-pm-wallet-encryption/src/waku.ts @@ -11,7 +11,7 @@ export const PrivateMessageContentTopic = '/eth-pm-wallet/1/private-message/proto'; export async function initWaku(): Promise { - const waku = await Waku.create({ bootstrap: true }); + const waku = await Waku.create({ bootstrap: { default: true } }); // Wait to be connected to at least one peer await new Promise((resolve, reject) => { diff --git a/examples/eth-pm/package-lock.json b/examples/eth-pm/package-lock.json index 209e3f9b93..8744098128 100644 --- a/examples/eth-pm/package-lock.json +++ b/examples/eth-pm/package-lock.json @@ -6440,9 +6440,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27967,9 +27967,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001237", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", - "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", diff --git a/examples/eth-pm/src/waku.ts b/examples/eth-pm/src/waku.ts index 8a3ba14c94..33e802bcef 100644 --- a/examples/eth-pm/src/waku.ts +++ b/examples/eth-pm/src/waku.ts @@ -9,7 +9,7 @@ export const PublicKeyContentTopic = '/eth-pm/1/public-key/proto'; export const PrivateMessageContentTopic = '/eth-pm/1/private-message/proto'; export async function initWaku(): Promise { - const waku = await Waku.create({ bootstrap: true }); + const waku = await Waku.create({ bootstrap: { default: true } }); // Wait to be connected to at least one peer await new Promise((resolve, reject) => { diff --git a/examples/relay-reactjs-chat/package-lock.json b/examples/relay-reactjs-chat/package-lock.json index 2a81ff4fc8..ccb965510f 100644 --- a/examples/relay-reactjs-chat/package-lock.json +++ b/examples/relay-reactjs-chat/package-lock.json @@ -4858,9 +4858,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -20062,9 +20062,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/examples/store-reactjs-chat/package-lock.json b/examples/store-reactjs-chat/package-lock.json index 0825849f05..4e4ad13e74 100644 --- a/examples/store-reactjs-chat/package-lock.json +++ b/examples/store-reactjs-chat/package-lock.json @@ -5383,9 +5383,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001298", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001298.tgz", - "integrity": "sha512-AcKqikjMLlvghZL/vfTHorlQsLDhGRalYf1+GmWCf5SCMziSGjRYQW/JEksj14NaYHIR6KIhrFAy0HV5C25UzQ==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -22888,9 +22888,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001298", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001298.tgz", - "integrity": "sha512-AcKqikjMLlvghZL/vfTHorlQsLDhGRalYf1+GmWCf5SCMziSGjRYQW/JEksj14NaYHIR6KIhrFAy0HV5C25UzQ==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/examples/web-chat/package-lock.json b/examples/web-chat/package-lock.json index dfb1a40e9d..7269753b47 100644 --- a/examples/web-chat/package-lock.json +++ b/examples/web-chat/package-lock.json @@ -6963,9 +6963,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -27344,9 +27344,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001296", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001296.tgz", - "integrity": "sha512-WfrtPEoNSoeATDlf4y3QvkwiELl9GyPLISV5GejTbbQRtQx4LhsXmc9IQ6XCL2d7UxCyEzToEZNMeqR79OUw8Q==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "case-sensitive-paths-webpack-plugin": { "version": "2.4.0", diff --git a/examples/web-chat/src/App.tsx b/examples/web-chat/src/App.tsx index ec42eda8eb..02a81caf75 100644 --- a/examples/web-chat/src/App.tsx +++ b/examples/web-chat/src/App.tsx @@ -1,6 +1,11 @@ import { useEffect, useReducer, useState } from 'react'; import './App.css'; -import { PageDirection, getBootstrapNodes, Waku, WakuMessage } from 'js-waku'; +import { + PageDirection, + getNodesFromHostedJson, + Waku, + WakuMessage, +} from 'js-waku'; import handleCommand from './command'; import Room from './Room'; import { WakuContext } from './WakuContext'; @@ -175,7 +180,9 @@ async function initWaku(setter: (waku: Waku) => void) { }, }, }, - bootstrap: getBootstrapNodes.bind({}, selectFleetEnv()), + bootstrap: { + getPeers: getNodesFromHostedJson.bind({}, selectFleetEnv()), + }, }); setter(waku); diff --git a/karma-live-data.conf.js b/karma-live-data.conf.js new file mode 100644 index 0000000000..4d2d777bac --- /dev/null +++ b/karma-live-data.conf.js @@ -0,0 +1,16 @@ +// import settings from default config file +let properties = null; +const originalConfigFn = require('./karma.conf.js'); +originalConfigFn({ + set: function (arg) { + properties = arg; + }, +}); + +// pass `--grep '[live data]'` to mocha to only run live data tests +properties.client.args = ['--grep', '[live data]]']; + +// export settings +module.exports = function (config) { + config.set(properties); +}; diff --git a/package-lock.json b/package-lock.json index 8d266c697a..f06589fdd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", @@ -2106,6 +2108,11 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "node_modules/@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -3118,6 +3125,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "node_modules/@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -4864,9 +4879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001251", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001251.tgz", - "integrity": "sha512-HOe1r+9VkU4TFmnU70z+r7OLmtR+/chB1rdcJUeQlAinjEeb0cKL20tlAtOagNZhbrtLnCvV19B4FmF1rgzl6A==", + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/browserslist" @@ -6183,6 +6198,42 @@ "receptacle": "^1.3.2" } }, + "node_modules/dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + }, + "bin": { + "dns-query": "bin/dns-query" + } + }, + "node_modules/dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "dependencies": { + "dns-packet": "^5.2.4" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -8368,6 +8419,11 @@ "node": ">= 8" } }, + "node_modules/hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -19349,6 +19405,11 @@ "chalk": "^4.0.0" } }, + "@leichtgewicht/ip-codec": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.3.tgz", + "integrity": "sha512-nkalE/f1RvRGChwBnEIoBfSEYOXnCRdleKuv6+lePbMDrMZXeDQnqak5XDOeBgrPPyPfAdcCu/B5z+v3VhplGg==" + }, "@motrix/nat-api": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@motrix/nat-api/-/nat-api-0.3.2.tgz", @@ -20233,6 +20294,14 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "@types/dns-packet": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.2.4.tgz", + "integrity": "sha512-OAruArypdNxR/tzbmrtoyEuXeNTLaZCpO19BXaNC10T5ACIbvjmvhmV2RDEy2eLc3w8IjK7SY3cvUCcAW+sfoQ==", + "requires": { + "@types/node": "*" + } + }, "@types/eslint": { "version": "7.28.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.1.tgz", @@ -21642,9 +21711,9 @@ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, "caniuse-lite": { - "version": "1.0.30001251", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001251.tgz", - "integrity": "sha512-HOe1r+9VkU4TFmnU70z+r7OLmtR+/chB1rdcJUeQlAinjEeb0cKL20tlAtOagNZhbrtLnCvV19B4FmF1rgzl6A==" + "version": "1.0.30001299", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001299.tgz", + "integrity": "sha512-iujN4+x7QzqA2NCSrS5VUy+4gLmRd4xv6vbBBsmfVqTx8bLAD8097euLqQgKxSVLvxjSDcvF1T/i9ocgnUFexw==" }, "capture-exit": { "version": "2.0.0", @@ -22697,6 +22766,33 @@ "receptacle": "^1.3.2" } }, + "dns-packet": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.3.1.tgz", + "integrity": "sha512-spBwIj0TK0Ey3666GwIdWVfUpLyubpU53BTCu8iPn4r4oXd9O14Hjg3EHw3ts2oed77/SeckunUYCyRlSngqHw==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.1" + } + }, + "dns-query": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dns-query/-/dns-query-0.8.0.tgz", + "integrity": "sha512-Gx3jYhdj9oLMZFieinpwpTFK0c2Q+teV53Se1+l4AbcWLPMUCBACu7qcj0IqTWwnpasWl8Gwgxeqw2RjoCwIoA==", + "requires": { + "@leichtgewicht/ip-codec": "^2.0.2", + "@types/dns-packet": "^5.2.0", + "dns-packet": "^5.3.0", + "dns-socket": "^4.2.2" + } + }, + "dns-socket": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-4.2.2.tgz", + "integrity": "sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==", + "requires": { + "dns-packet": "^5.2.4" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -24357,6 +24453,11 @@ } } }, + "hi-base32": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/hi-base32/-/hi-base32-0.5.1.tgz", + "integrity": "sha512-EmBBpvdYh/4XxsnUybsPag6VikPYnN30td+vQk+GI3qpahVEG9+gTkG0aXVxTjBqQ5T6ijbWIu77O+C5WFWsnA==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", diff --git a/package.json b/package.json index 7202a433f4..3480f3333c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "pretest": "run-s pretest:*", "pretest:1-init-git-submodules": "[ -f './nim-waku/build/wakunode2' ] || git submodule update --init --recursive", "pretest:2-build-nim-waku": "[ -f './nim-waku/build/wakunode2' ] || run-s nim-waku:build", - "examples:pretest": "for d in examples/*; do (cd $d; npm install); done", + "examples:pretest": "for d in examples/*/; do (cd $d && npm install); done", "nim-waku:build": "(cd nim-waku; NIMFLAGS=\"-d:chronicles_colors=off -d:chronicles_sinks=textlines -d:chronicles_log_level=TRACE\" make -j$(nproc --all 2>/dev/null || echo 2) wakunode2)", "nim-waku:force-build": "(cd nim-waku && rm -rf ./build/ ./vendor && make -j$(nproc --all 2>/dev/null || echo 2) update) && run-s nim-waku:build", "test": "run-s build test:*", @@ -39,7 +39,8 @@ "test:spelling": "cspell \"{README.md,.github/*.md,guides/*.md,src/**/*.ts}\"", "test:unit": "nyc --silent mocha", "test:karma": "karma start", - "examples:test": "run-s examples:pretest; for d in examples/*; do (cd $d; npm test;); done", + "test:karma-live-data": "LIVE_DATA_TESTS=true karma start ./karma-live-data.conf.js", + "examples:test": "run-s examples:pretest; for d in examples/*/; do (cd $d && npm test;); done", "proto": "run-s proto:*", "proto:lint": "buf lint", "proto:build": "buf generate", @@ -58,7 +59,9 @@ "base64url": "^3.0.1", "bigint-buffer": "^1.1.5", "debug": "^4.3.1", + "dns-query": "^0.8.0", "ecies-geth": "^1.5.2", + "hi-base32": "^0.5.1", "it-concat": "^2.0.0", "it-length-prefixed": "^5.0.2", "js-sha3": "^0.8.0", diff --git a/src/index.ts b/src/index.ts index 435ec291ae..d3a72319e6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,7 @@ -export { getBootstrapNodes } from './lib/discovery'; +export { getNodesFromHostedJson } from './lib/discovery'; +export * as discovery from './lib/discovery'; + +export * as enr from './lib/enr'; export * as utils from './lib/utils'; diff --git a/src/lib/discovery/bootstrap.ts b/src/lib/discovery/bootstrap.ts new file mode 100644 index 0000000000..771eaad558 --- /dev/null +++ b/src/lib/discovery/bootstrap.ts @@ -0,0 +1,112 @@ +import debug from 'debug'; +import { Multiaddr } from 'multiaddr'; + +import { DnsNodeDiscovery } from './dns'; + +import { getNodesFromHostedJson, getPseudoRandomSubset } from './index'; + +const dbg = debug('waku:discovery:bootstrap'); + +/** + * Setup discovery method used to bootstrap. + * + * Only one method is used. `default`, `peers`, `getPeers` and `enr` options are mutually exclusive. + */ +export interface BootstrapOptions { + /** + * The maximum of peers to connect to as part of the bootstrap process. + * + * @default [[Bootstrap.DefaultMaxPeers]] + */ + maxPeers?: number; + /** + * Use the default discovery method. Overrides all other options but `maxPeers` + * + * The default discovery method is likely to change overtime as new discovery + * methods are implemented. + * + * @default false + */ + default?: boolean; + /** + * Multiaddrs of peers to connect to. + */ + peers?: string[]; + /** + * Getter that retrieve multiaddrs of peers to connect to. + */ + getPeers?: () => Promise; + /** + * An EIP-1459 ENR Tree URL. For example: + * "enrtree://AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C@test.nodes.vac.dev" + */ + enrUrl?: string; +} + +/** + * Parse options and expose function to return bootstrap peer addresses. + */ +export class Bootstrap { + public static DefaultMaxPeers = 1; + + public readonly getBootstrapPeers: (() => Promise) | undefined; + + constructor(opts: BootstrapOptions) { + const maxPeers = opts.maxPeers ?? Bootstrap.DefaultMaxPeers; + + if (opts.default) { + dbg('Bootstrap: Use hosted list of peers.'); + + this.getBootstrapPeers = getNodesFromHostedJson.bind( + {}, + undefined, + undefined, + maxPeers + ); + } else if (opts.peers !== undefined && opts.peers.length > 0) { + dbg('Bootstrap: Use provided list of peers.'); + + const allPeers: Multiaddr[] = opts.peers.map( + (node: string) => new Multiaddr(node) + ); + const peers = getPseudoRandomSubset(allPeers, maxPeers); + this.getBootstrapPeers = (): Promise => + Promise.resolve(peers); + } else if (typeof opts.getPeers === 'function') { + dbg('Bootstrap: Use provided getPeers function.'); + const getPeers = opts.getPeers; + + this.getBootstrapPeers = async (): Promise => { + const allPeers = await getPeers(); + return getPseudoRandomSubset( + allPeers, + maxPeers + ).map((node) => new Multiaddr(node)); + }; + } else if (opts.enrUrl) { + const enrUrl = opts.enrUrl; + dbg('Bootstrap: Use provided EIP-1459 ENR Tree URL.'); + + const dns = DnsNodeDiscovery.dnsOverHttp(); + + this.getBootstrapPeers = async (): Promise => { + const enrs = await dns.getPeers(maxPeers, [enrUrl]); + const addresses: Multiaddr[] = []; + enrs.forEach((enr) => { + if (!enr.multiaddrs) return; + + enr.multiaddrs.forEach((ma: Multiaddr) => { + // Only return secure websocket addresses + if (ma.protoNames().includes('wss')) { + addresses.push(ma); + } + }); + }); + return addresses; + }; + } else { + dbg('No bootstrap method specified, no peer will be returned'); + this.getBootstrapPeers = undefined; + } + } +} diff --git a/src/lib/discovery/dns.spec.ts b/src/lib/discovery/dns.spec.ts new file mode 100644 index 0000000000..964b36d457 --- /dev/null +++ b/src/lib/discovery/dns.spec.ts @@ -0,0 +1,203 @@ +import { expect } from 'chai'; + +import { DnsClient, DnsNodeDiscovery } from './dns'; +import testData from './testdata.json'; + +const mockData = testData.dns; + +const host = 'nodes.example.org'; +const rootDomain = 'JORXBYVVM7AEKETX5DGXW44EAY'; +const branchDomainA = 'D2SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainB = 'D3SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainC = 'D4SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainD = 'D5SNLTAGWNQ34NTQTPHNZDECFU'; +const partialBranchA = 'AAAA'; +const partialBranchB = 'BBBB'; +const singleBranch = `enrtree-branch:${branchDomainA}`; +const doubleBranch = `enrtree-branch:${branchDomainA},${branchDomainB}`; +const multiComponentBranch = [ + `enrtree-branch:${branchDomainA},${partialBranchA}`, + `${partialBranchB},${branchDomainB}`, +]; + +// Note: once td.when is asked to throw for an input it will always throw. +// Input can't be re-used for a passing case. +const errorBranchA = `enrtree-branch:${branchDomainC}`; +const errorBranchB = `enrtree-branch:${branchDomainD}`; + +/** + * Mocks DNS resolution. + */ +class MockDNS implements DnsClient { + fqdnRes: Map; + fqdnThrows: string[]; + + constructor() { + this.fqdnRes = new Map(); + this.fqdnThrows = []; + } + + addRes(fqdn: string, res: string[]): void { + this.fqdnRes.set(fqdn, res); + } + + addThrow(fqdn: string): void { + this.fqdnThrows.push(fqdn); + } + + resolveTXT(fqdn: string): Promise { + if (this.fqdnThrows.includes(fqdn)) throw 'Mock DNS throws.'; + + const res = this.fqdnRes.get(fqdn); + + if (!res) throw `Mock DNS could not resolve ${fqdn}`; + + return Promise.resolve(res); + } +} + +describe('DNS Node Discovery', () => { + let mockDns: MockDNS; + + beforeEach(() => { + mockDns = new MockDNS(); + mockDns.addRes(host, [mockData.enrRoot]); + }); + + it('retrieves a single peer', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(1); + expect(peers[0].ip).to.eq('45.77.40.127'); + expect(peers[0].tcp).to.eq(30303); + }); + + it('retrieves all peers (2) when maxQuantity larger than DNS tree size', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [doubleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrB]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(2); + expect(peers[0].ip).to.not.eq(peers[1].ip); + }); + + it('retrieves all peers (3) when branch entries are composed of multiple strings', async function () { + mockDns.addRes(`${rootDomain}.${host}`, multiComponentBranch); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enr]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrA]); + mockDns.addRes(`${partialBranchA}${partialBranchB}.${host}`, [ + mockData.enrB, + ]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(3); + expect(peers[0].ip).to.not.eq(peers[1].ip); + expect(peers[0].ip).to.not.eq(peers[2].ip); + expect(peers[1].ip).to.not.eq(peers[2].ip); + }); + + it('it tolerates circular branch references', async function () { + // root --> branchA + // branchA --> branchA + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [singleBranch]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + }); + + it('recovers when dns.resolve returns empty', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + + // Empty response case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + let peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + + // No TXT records case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores domain fetching errors', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchA]); + mockDns.addThrow(`${branchDomainC}.${host}`); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores unrecognized TXT record formats', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [mockData.enrBranchBadPrefix]); + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('caches peers it previously fetched', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchB]); + mockDns.addRes(`${branchDomainD}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DnsNodeDiscovery(mockDns); + const peersA = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersA.length).to.eq(1); + + // Specify that a subsequent network call retrieving the same peer should throw. + // This test passes only if the peer is fetched from cache + mockDns.addThrow(`${branchDomainD}.${host}`); + + const peersB = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersB.length).to.eq(1); + expect(peersA[0].ip).to.eq(peersB[0].ip); + }); +}); + +describe('DNS Node Discovery [live data]', function () { + const publicKey = 'AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C'; + const fqdn = 'test.nodes.vac.dev'; + const enrTree = `enrtree://${publicKey}@${fqdn}`; + const ipTestRegex = /^\d+\.\d+\.\d+\.\d+$/; + const maxQuantity = 3; + + before(function () { + if (process.env.CI && !process.env.LIVE_DATA_TESTS) { + this.skip(); + } + }); + + it(`should retrieve ${maxQuantity} PeerInfos for test.nodes.vac.dev`, async function () { + this.timeout(5000); + // Google's dns server address. Needs to be set explicitly to run in CI + const dnsNodeDiscovery = DnsNodeDiscovery.dnsOverHttp(); + const peers = await dnsNodeDiscovery.getPeers(maxQuantity, [enrTree]); + + expect(peers.length).to.eq(maxQuantity); + + // TODO: Test multiaddrs entry + console.log(peers.map((peer) => peer.multiaddrs)); + + const seen: string[] = []; + for (const peer of peers) { + expect(peer!.ip!).to.match(ipTestRegex); + expect(seen).to.not.include(peer!.ip!); + seen.push(peer!.ip!); + } + }); +}); diff --git a/src/lib/discovery/dns.ts b/src/lib/discovery/dns.ts new file mode 100644 index 0000000000..34f4b2b10a --- /dev/null +++ b/src/lib/discovery/dns.ts @@ -0,0 +1,201 @@ +import assert from 'assert'; + +import { debug } from 'debug'; + +import { ENR } from '../enr'; + +import { DnsOverHttps, Endpoints } from './dns_over_https'; +import { ENRTree } from './enrtree'; + +const dbg = debug('waku:discovery:dns'); + +export type SearchContext = { + domain: string; + publicKey: string; + visits: { [key: string]: boolean }; +}; + +export interface DnsClient { + resolveTXT: (domain: string) => Promise; +} + +export class DnsNodeDiscovery { + private readonly dns: DnsClient; + private readonly _DNSTreeCache: { [key: string]: string }; + private readonly _errorTolerance: number = 10; + + public static dnsOverHttp(endpoints?: Endpoints): DnsNodeDiscovery { + const dnsClient = new DnsOverHttps(endpoints); + return new DnsNodeDiscovery(dnsClient); + } + + /** + * Returns a list of verified peers listed in an EIP-1459 DNS tree. Method may + * return fewer peers than requested if `maxQuantity` is larger than the number + * of ENR records or the number of errors/duplicate peers encountered by randomized + * search exceeds `maxQuantity` plus the `errorTolerance` factor. + */ + async getPeers(maxQuantity: number, enrTreeUrls: string[]): Promise { + let totalSearches = 0; + const peers: ENR[] = []; + + const networkIndex = Math.floor(Math.random() * enrTreeUrls.length); + const { publicKey, domain } = ENRTree.parseTree(enrTreeUrls[networkIndex]); + + while ( + peers.length < maxQuantity && + totalSearches < maxQuantity + this._errorTolerance + ) { + const context: SearchContext = { + domain, + publicKey, + visits: {}, + }; + + const peer = await this._search(domain, context); + + if (peer && isNewPeer(peer, peers)) { + peers.push(peer); + dbg( + `got new peer candidate from DNS address=${peer.nodeId}@${peer.ip}` + ); + } + + totalSearches++; + } + return peers; + } + + public constructor(dns: DnsClient) { + this._DNSTreeCache = {}; + this.dns = dns; + } + + /** + * Runs a recursive, randomized descent of the DNS tree to retrieve a single + * ENR record as an ENR. Returns null if parsing or DNS resolution fails. + */ + private async _search( + subdomain: string, + context: SearchContext + ): Promise { + const entry = await this._getTXTRecord(subdomain, context); + context.visits[subdomain] = true; + + let next: string; + let branches: string[]; + + const entryType = getEntryType(entry); + try { + switch (entryType) { + case ENRTree.ROOT_PREFIX: + next = ENRTree.parseAndVerifyRoot(entry, context.publicKey); + return await this._search(next, context); + case ENRTree.BRANCH_PREFIX: + branches = ENRTree.parseBranch(entry); + next = selectRandomPath(branches, context); + return await this._search(next, context); + case ENRTree.RECORD_PREFIX: + return ENR.decodeTxt(entry); + default: + return null; + } + } catch (error) { + dbg( + `Failed to search DNS tree ${entryType} at subdomain ${subdomain}: ${error}` + ); + return null; + } + } + + /** + * Retrieves the TXT record stored at a location from either + * this DNS tree cache or via DNS query + */ + private async _getTXTRecord( + subdomain: string, + context: SearchContext + ): Promise { + if (this._DNSTreeCache[subdomain]) { + return this._DNSTreeCache[subdomain]; + } + + // Location is either the top level tree entry host or a subdomain of it. + const location = + subdomain !== context.domain + ? `${subdomain}.${context.domain}` + : context.domain; + + const response = await this.dns.resolveTXT(location); + + assert( + response.length, + 'Received empty result array while fetching TXT record' + ); + assert(response[0].length, 'Received empty TXT record'); + + // Branch entries can be an array of strings of comma delimited subdomains, with + // some subdomain strings split across the array elements + const result = response.join(''); + + this._DNSTreeCache[subdomain] = result; + return result; + } +} + +function getEntryType(entry: string): string { + if (entry.startsWith(ENRTree.ROOT_PREFIX)) return ENRTree.ROOT_PREFIX; + if (entry.startsWith(ENRTree.BRANCH_PREFIX)) return ENRTree.BRANCH_PREFIX; + if (entry.startsWith(ENRTree.RECORD_PREFIX)) return ENRTree.RECORD_PREFIX; + + return ''; +} + +/** + * Returns a randomly selected subdomain string from the list provided by a branch + * entry record. + * + * The client must track subdomains which are already resolved to avoid + * going into an infinite loop b/c branch entries can contain + * circular references. It’s in the client’s best interest to traverse the + * tree in random order. + */ +function selectRandomPath(branches: string[], context: SearchContext): string { + // Identify domains already visited in this traversal of the DNS tree. + // Then filter against them to prevent cycles. + const circularRefs: { [key: number]: boolean } = {}; + for (const [idx, subdomain] of branches.entries()) { + if (context.visits[subdomain]) { + circularRefs[idx] = true; + } + } + // If all possible paths are circular... + if (Object.keys(circularRefs).length === branches.length) { + throw new Error('Unresolvable circular path detected'); + } + + // Randomly select a viable path + let index; + do { + index = Math.floor(Math.random() * branches.length); + } while (circularRefs[index]); + + return branches[index]; +} + +/** + * @returns false if candidate peer already exists in the + * current collection of peers based on the node id value; + * true otherwise. + */ +function isNewPeer(peer: ENR | null, peers: ENR[]): boolean { + if (!peer || !peer.nodeId) return false; + + for (const existingPeer of peers) { + if (peer.nodeId === existingPeer.nodeId) { + return false; + } + } + + return true; +} diff --git a/src/lib/discovery/dns_over_https.ts b/src/lib/discovery/dns_over_https.ts new file mode 100644 index 0000000000..9f0b33e70f --- /dev/null +++ b/src/lib/discovery/dns_over_https.ts @@ -0,0 +1,61 @@ +import { TxtAnswer } from 'dns-packet'; +import { + endpoints as defaultEndpoints, + Endpoint, + EndpointProps, + query, +} from 'dns-query'; + +import { DnsClient } from './dns'; + +const { cloudflare, google, opendns } = defaultEndpoints; + +export type Endpoints = + | 'doh' + | 'dns' + | Iterable; + +export class DnsOverHttps implements DnsClient { + /** + * Create new Dns-Over-Http DNS client. + * + * @param endpoints The endpoints for Dns-Over-Https queries. + * See [dns-query](https://www.npmjs.com/package/dns-query) for details. + * Defaults to cloudflare, google and opendns. + * + * @throws {code: string} If DNS query fails. + */ + public constructor( + public endpoints: Endpoints = [cloudflare, google, opendns] + ) {} + + async resolveTXT(domain: string): Promise { + const response = await query({ + questions: [{ type: 'TXT', name: domain }], + }); + + const answers = response.answers as TxtAnswer[]; + + const data = answers.map((a) => a.data); + + const result: string[] = []; + + data.forEach((d) => { + if (typeof d === 'string') { + result.push(d); + } else if (Array.isArray(d)) { + d.forEach((sd) => { + if (typeof sd === 'string') { + result.push(sd); + } else { + result.push(Buffer.from(sd).toString('utf-8')); + } + }); + } else { + result.push(Buffer.from(d).toString('utf-8')); + } + }); + + return result; + } +} diff --git a/src/lib/discovery/enrtree.spec.ts b/src/lib/discovery/enrtree.spec.ts new file mode 100644 index 0000000000..af7402b82e --- /dev/null +++ b/src/lib/discovery/enrtree.spec.ts @@ -0,0 +1,89 @@ +import { expect } from 'chai'; + +import { ENRTree } from './enrtree'; +import testData from './testdata.json'; + +const dns = testData.dns; + +describe('ENRTree', () => { + // Root DNS entries + it('ENRTree (root): should parse and verify and DNS root entry', () => { + const subdomain = ENRTree.parseAndVerifyRoot(dns.enrRoot, dns.publicKey); + + expect(subdomain).to.eq('JORXBYVVM7AEKETX5DGXW44EAY'); + }); + + it('ENRTree (root): should error if DNS root entry is mis-prefixed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadPrefix, dns.publicKey); + } catch (e) { + expect(e.toString()).includes( + "ENRTree root entry must start with 'enrtree-root:'" + ); + } + }); + + it('ENRTree (root): should error if DNS root entry signature is invalid', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootBadSig, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Unable to verify ENRTree root signature'); + } + }); + + it('ENRTree (root): should error if DNS root entry is malformed', () => { + try { + ENRTree.parseAndVerifyRoot(dns.enrRootMalformed, dns.publicKey); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree root entry'); + } + }); + + // Tree DNS entries + it('ENRTree (tree): should parse a DNS tree entry', () => { + const { publicKey, domain } = ENRTree.parseTree(dns.enrTree); + + expect(publicKey).to.eq(dns.publicKey); + expect(domain).to.eq('nodes.example.org'); + }); + + it('ENRTree (tree): should error if DNS tree entry is mis-prefixed', () => { + try { + ENRTree.parseTree(dns.enrTreeBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree tree entry must start with 'enrtree:'" + ); + } + }); + + it('ENRTree (tree): should error if DNS tree entry is misformatted', () => { + try { + ENRTree.parseTree(dns.enrTreeMalformed); + } catch (e) { + expect(e.toString()).includes('Could not parse ENRTree tree entry'); + } + }); + + // Branch entries + it('ENRTree (branch): should parse and verify a single component DNS branch entry', () => { + const expected = [ + 'D2SNLTAGWNQ34NTQTPHNZDECFU', + '67BLTJEU5R2D5S3B4QKJSBRFCY', + 'A2HDMZBB4JIU53VTEGC4TG6P4A', + ]; + + const branches = ENRTree.parseBranch(dns.enrBranch); + expect(branches).to.deep.eq(expected); + }); + + it('ENRTree (branch): should error if DNS branch entry is mis-prefixed', () => { + try { + ENRTree.parseBranch(dns.enrBranchBadPrefix); + } catch (e) { + expect(e.toString()).includes( + "ENRTree branch entry must start with 'enrtree-branch:'" + ); + } + }); +}); diff --git a/src/lib/discovery/enrtree.ts b/src/lib/discovery/enrtree.ts new file mode 100644 index 0000000000..718dad922b --- /dev/null +++ b/src/lib/discovery/enrtree.ts @@ -0,0 +1,116 @@ +import assert from 'assert'; + +import base64url from 'base64url'; +import * as base32 from 'hi-base32'; +import { ecdsaVerify } from 'secp256k1'; + +import { ENR } from '../enr'; +import { keccak256Buf } from '../utils'; + +export type ENRRootValues = { + eRoot: string; + lRoot: string; + seq: number; + signature: string; +}; + +export type ENRTreeValues = { + publicKey: string; + domain: string; +}; + +export class ENRTree { + public static readonly RECORD_PREFIX = ENR.RECORD_PREFIX; + public static readonly TREE_PREFIX = 'enrtree:'; + public static readonly BRANCH_PREFIX = 'enrtree-branch:'; + public static readonly ROOT_PREFIX = 'enrtree-root:'; + + /** + * Extracts the branch subdomain referenced by a DNS tree root string after verifying + * the root record signature with its base32 compressed public key. + */ + static parseAndVerifyRoot(root: string, publicKey: string): string { + assert( + root.startsWith(this.ROOT_PREFIX), + `ENRTree root entry must start with '${this.ROOT_PREFIX}'` + ); + + const rootValues = ENRTree.parseRootValues(root); + const decodedPublicKey = base32.decode.asBytes(publicKey); + + // The signature is a 65-byte secp256k1 over the keccak256 hash + // of the record content, excluding the `sig=` part, encoded as URL-safe base64 string + // (Trailing recovery bit must be trimmed to pass `ecdsaVerify` method) + const signedComponent = root.split(' sig')[0]; + const signedComponentBuffer = Buffer.from(signedComponent); + const signatureBuffer = base64url + .toBuffer(rootValues.signature) + .slice(0, 64); + const keyBuffer = Buffer.from(decodedPublicKey); + + const isVerified = ecdsaVerify( + signatureBuffer, + keccak256Buf(signedComponentBuffer), + keyBuffer + ); + + assert(isVerified, 'Unable to verify ENRTree root signature'); + + return rootValues.eRoot; + } + + static parseRootValues(txt: string): ENRRootValues { + const matches = txt.match( + /^enrtree-root:v1 e=([^ ]+) l=([^ ]+) seq=(\d+) sig=([^ ]+)$/ + ); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree root entry'); + + matches.shift(); // The first entry is the full match + const [eRoot, lRoot, seq, signature] = matches; + + assert.ok(eRoot, "Could not parse 'e' value from ENRTree root entry"); + assert.ok(lRoot, "Could not parse 'l' value from ENRTree root entry"); + assert.ok(seq, "Could not parse 'seq' value from ENRTree root entry"); + assert.ok(signature, "Could not parse 'sig' value from ENRTree root entry"); + + return { eRoot, lRoot, seq: Number(seq), signature }; + } + + /** + * Returns the public key and top level domain of an ENR tree entry. + * The domain is the starting point for traversing a set of linked DNS TXT records + * and the public key is used to verify the root entry record + */ + static parseTree(tree: string): ENRTreeValues { + assert( + tree.startsWith(this.TREE_PREFIX), + `ENRTree tree entry must start with '${this.TREE_PREFIX}'` + ); + + const matches = tree.match(/^enrtree:\/\/([^@]+)@(.+)$/); + + assert.ok(Array.isArray(matches), 'Could not parse ENRTree tree entry'); + + matches.shift(); // The first entry is the full match + const [publicKey, domain] = matches; + + assert.ok(publicKey, 'Could not parse public key from ENRTree tree entry'); + assert.ok(domain, 'Could not parse domain from ENRTree tree entry'); + + return { publicKey, domain }; + } + + /** + * Returns subdomains listed in an ENR branch entry. These in turn lead to + * either further branch entries or ENR records. + */ + static parseBranch(branch: string): string[] { + assert( + branch.startsWith(this.BRANCH_PREFIX), + `ENRTree branch entry must start with '${this.BRANCH_PREFIX}'` + ); + + return branch.split(this.BRANCH_PREFIX)[1].split(','); + } +} diff --git a/src/lib/discovery.ts b/src/lib/discovery/hosted_json.ts similarity index 80% rename from src/lib/discovery.ts rename to src/lib/discovery/hosted_json.ts index 6fabcf5fa6..111675e20d 100644 --- a/src/lib/discovery.ts +++ b/src/lib/discovery/hosted_json.ts @@ -1,11 +1,3 @@ -import axios from 'axios'; -import debug from 'debug'; -import { shuffle } from 'libp2p-gossipsub/src/utils'; - -const dbg = debug('waku:discovery'); - -const DefaultWantedNumber = 1; - /** * GET list of nodes from remote HTTP host. * @@ -23,11 +15,20 @@ const DefaultWantedNumber = 1; * @throws If the remote host is unreachable or the response cannot be parsed * according to the passed _path_. */ -export async function getBootstrapNodes( +import axios from 'axios'; +import debug from 'debug'; +import { Multiaddr } from 'multiaddr'; + +import { getPseudoRandomSubset } from './index'; +const dbg = debug('waku:discovery'); + +const DefaultWantedNumber = 1; + +export async function getNodesFromHostedJson( path: string[] = ['fleets', 'wakuv2.prod', 'waku-websocket'], url = 'https://fleets.status.im/', wantedNumber: number = DefaultWantedNumber -): Promise { +): Promise { if (wantedNumber <= 0) { return []; } @@ -52,15 +53,18 @@ export async function getBootstrapNodes( } if (Array.isArray(nodes)) { - return getPseudoRandomSubset(nodes, wantedNumber); + return getPseudoRandomSubset(nodes, wantedNumber).map( + (node: string) => new Multiaddr(node) + ); } if (typeof nodes === 'string') { - return [nodes]; + return [new Multiaddr(nodes)]; } if (typeof nodes === 'object') { - nodes = Object.values(nodes); + nodes = Object.values(nodes) as string[]; + nodes = nodes.map((node: string) => new Multiaddr(node)); return getPseudoRandomSubset(nodes, wantedNumber); } @@ -68,14 +72,3 @@ export async function getBootstrapNodes( nodes )}`; } - -export function getPseudoRandomSubset( - values: string[], - wantedNumber: number -): string[] { - if (values.length <= wantedNumber) { - return values; - } - - return shuffle(values).slice(0, wantedNumber); -} diff --git a/src/lib/discovery.spec.ts b/src/lib/discovery/index.spec.ts similarity index 94% rename from src/lib/discovery.spec.ts rename to src/lib/discovery/index.spec.ts index ff4b7a7fa2..0969766e95 100644 --- a/src/lib/discovery.spec.ts +++ b/src/lib/discovery/index.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; -import { getPseudoRandomSubset } from './discovery'; +import { getPseudoRandomSubset } from './index'; describe('Discovery', () => { it('returns all values when wanted number matches available values', function () { diff --git a/src/lib/discovery/index.ts b/src/lib/discovery/index.ts new file mode 100644 index 0000000000..5ca9251fa9 --- /dev/null +++ b/src/lib/discovery/index.ts @@ -0,0 +1,18 @@ +import { shuffle } from 'libp2p-gossipsub/src/utils'; + +export { getNodesFromHostedJson } from './hosted_json'; +export { Bootstrap, BootstrapOptions } from './bootstrap'; +export { DnsClient, DnsNodeDiscovery, SearchContext } from './dns'; +export { Endpoints, DnsOverHttps } from './dns_over_https'; +export { ENRTree, ENRTreeValues, ENRRootValues } from './enrtree'; + +export function getPseudoRandomSubset( + values: T[], + wantedNumber: number +): T[] { + if (values.length <= wantedNumber) { + return values; + } + + return shuffle(values).slice(0, wantedNumber); +} diff --git a/src/lib/discovery/testdata.json b/src/lib/discovery/testdata.json new file mode 100644 index 0000000000..86a3a7b1be --- /dev/null +++ b/src/lib/discovery/testdata.json @@ -0,0 +1,18 @@ +{ + "dns": { + "publicKey": "AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE", + "enr": "enr:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrA": "enr:-Jq4QAopXcF_SSfOwl_AmLdrMUnHQO1Rx-XV4gYeySSK32PTbQ8volkh3IQy1ag1Gkl6O-C5rjskj3EyDi8XVzck4PMVg2V0aMrJhKALwySDbxWAgmlkgnY0gmlwhC1NKH-Jc2VjcDI1NmsxoQO5wMEjJLtqT-h6zhef0xsO-SW-pcQD-yuNqCr3GTEZFoN0Y3CCdl-DdWRwgnZf", + "enrB": "enr:-Je4QAFx_6rFjCxCLPUbxIA_KS7FhCYeTU6fXmbj1V08f8DPCUAB9bLoY2Yy7q2hIEby7Yf6e_v7gbofloB1oTnjqeYDg2V0aMfGhOAp6ZGAgmlkgnY0gmlwhLxf-D2Jc2VjcDI1NmsxoQOou7vgUXL96E5CzBsCE6N1GSMqlAACtUxRiNpq6vnB6IN0Y3CCdl-DdWRwgnZf", + "enrRoot": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranch": "enrtree-branch:D2SNLTAGWNQ34NTQTPHNZDECFU,67BLTJEU5R2D5S3B4QKJSBRFCY,A2HDMZBB4JIU53VTEGC4TG6P4A", + "enrTree": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@nodes.example.org", + "enrBadPrefix": "enrabc:-Je4QA1w6JNgH44256YxSTujRYIIy-oeCzL3tIvCIIHEZ_HgWbbFlrtfghWaGKQA9PH2INlnOGiKAU66hhVEoocrZdo0g2V0aMfGhOAp6ZGAgmlkgnY0gmlwhChxb4eJc2VjcDI1NmsxoQMla1-eA4bdHAeDEGv_z115bE16iA4GxcbGd-OlmKnSpYN0Y3CCdl-DdWRwgnZf", + "enrRootBadPrefix": "enrtree:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrBranchBadPrefix": "Z64M,JOECK7UUYUFVX24QGXYLR3UHDU,RR6SC4GUZBKLFA2WO4IUY6YGEE,EQRME5EAOS7AJHHLDDZNDYT7GI,JXHUMLDSGKU6UQWYFMNCFYQFHQ,4SNDLPNM3CBG2KLBMRSTHWFNP4,WEEEFCKUXOGU4QPKCRBBEHQLEY,CPXM5AOSTICZ3TODJFQACGBWMU,7U26GD37NS6DV72PDAURZI4WUY,MYLQIGMR5GTKPPBMXIINZ2ALGU", + "enrTreeBadPrefix": "entree-branch://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@nodes.example.org", + "enrRootBadSig": "enrtree-root:v1 e=JORXBYVVM7AEKETX5DGXW44EAY l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Aa7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrRootMalformed": "enrtree-root:v1 e=FDXN3SN67NA5DKA4J2GOK7BVQI seq=1839 sig=Ma7yIqW2gj59dY8F6plfL7dfotaBPz285mu_XZK1e5VRzNrnf0pCAfacu4fBLuE7jMX-nDbqCM1sFiWWLq8WogE", + "enrTreeMalformed": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2nodes.example.org" + } +} diff --git a/src/lib/enr/enr.spec.ts b/src/lib/enr/enr.spec.ts index e9e4d133c5..c7feb48610 100644 --- a/src/lib/enr/enr.spec.ts +++ b/src/lib/enr/enr.spec.ts @@ -113,7 +113,7 @@ describe('ENR', function () { ENR.decodeTxt(txt); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); }); @@ -153,7 +153,7 @@ describe('ENR', function () { enr.verify(Buffer.alloc(0), Buffer.alloc(0)); assert.fail('Expect error here'); } catch (err) { - expect(err.message).to.be.equal('Failed to verify enr: No public key'); + expect(err.message).to.be.equal('Failed to verify ENR: No public key'); } }); diff --git a/src/lib/enr/enr.ts b/src/lib/enr/enr.ts index 440ca16ba2..d41e6584b2 100644 --- a/src/lib/enr/enr.ts +++ b/src/lib/enr/enr.ts @@ -25,6 +25,7 @@ import { ENRKey, ENRValue, NodeId, SequenceNumber } from './types'; import * as v4 from './v4'; export class ENR extends Map { + public static readonly RECORD_PREFIX = 'enr:'; public seq: SequenceNumber; public signature: Buffer | null; @@ -93,8 +94,10 @@ export class ENR extends Map { } static decodeTxt(encoded: string): ENR { - if (!encoded.startsWith('enr:')) { - throw new Error("string encoded ENR must start with 'enr:'"); + if (!encoded.startsWith(this.RECORD_PREFIX)) { + throw new Error( + `"string encoded ENR must start with '${this.RECORD_PREFIX}'` + ); } return ENR.decode(base64url.toBuffer(encoded.slice(4))); } @@ -388,6 +391,7 @@ export class ENR extends Map { return new Multiaddr(maBuf); } + setLocationMultiaddr(multiaddr: Multiaddr): void { const protoNames = multiaddr.protoNames(); if ( @@ -428,7 +432,7 @@ export class ENR extends Map { throw new Error(ERR_INVALID_ID); } if (!this.publicKey) { - throw new Error('Failed to verify enr: No public key'); + throw new Error('Failed to verify ENR: No public key'); } return v4.verify(this.publicKey, data, signature); } @@ -471,6 +475,6 @@ export class ENR extends Map { } encodeTxt(privateKey?: Buffer): string { - return 'enr:' + base64url.encode(this.encode(privateKey)); + return ENR.RECORD_PREFIX + base64url.encode(this.encode(privateKey)); } } diff --git a/src/lib/enr/index.ts b/src/lib/enr/index.ts index 79f8330afc..8dc9b8c077 100644 --- a/src/lib/enr/index.ts +++ b/src/lib/enr/index.ts @@ -4,3 +4,4 @@ export * from './constants'; export * from './enr'; export * from './types'; export * from './create'; +export * from './keypair'; diff --git a/src/lib/enr/v4.ts b/src/lib/enr/v4.ts index eae00be7f0..7024541b75 100644 --- a/src/lib/enr/v4.ts +++ b/src/lib/enr/v4.ts @@ -1,4 +1,3 @@ -import { Buffer } from 'buffer'; import crypto from 'crypto'; import { keccak256 } from 'js-sha3'; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 605e5b95dc..398e450c31 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,3 +1,5 @@ +import { keccak256, Message } from 'js-sha3'; + export function hexToBuf(hex: string | Buffer | Uint8Array): Buffer { if (typeof hex === 'string') { return Buffer.from(hex.replace(/^0x/i, ''), 'hex'); @@ -31,3 +33,7 @@ export function equalByteArrays( return aBuf.compare(bBuf) === 0; } + +export function keccak256Buf(message: Message): Buffer { + return Buffer.from(keccak256.arrayBuffer(message)); +} diff --git a/src/lib/waku.node.spec.ts b/src/lib/waku.node.spec.ts index ce50591170..4e2a3cffd5 100644 --- a/src/lib/waku.node.spec.ts +++ b/src/lib/waku.node.spec.ts @@ -38,7 +38,7 @@ describe('Waku Dial [node only]', function () { waku = await Waku.create({ staticNoiseKey: NOISE_KEY_1, - bootstrap: true, + bootstrap: { default: true }, }); const connectedPeerID: PeerId = await new Promise((resolve) => { @@ -68,7 +68,7 @@ describe('Waku Dial [node only]', function () { libp2p: { modules: { transport: [TCP] }, }, - bootstrap: [multiAddrWithId], + bootstrap: { peers: [multiAddrWithId] }, }); const connectedPeerID: PeerId = await new Promise((resolve) => { @@ -102,8 +102,10 @@ describe('Waku Dial [node only]', function () { libp2p: { modules: { transport: [TCP] }, }, - bootstrap: () => { - return [multiAddrWithId]; + bootstrap: { + getPeers: async () => { + return [multiAddrWithId]; + }, }, }); diff --git a/src/lib/waku.ts b/src/lib/waku.ts index 6900ab6e07..977b07c19b 100644 --- a/src/lib/waku.ts +++ b/src/lib/waku.ts @@ -2,7 +2,7 @@ import { bytes } from '@chainsafe/libp2p-noise/dist/src/@types/basic'; import { Noise } from '@chainsafe/libp2p-noise/dist/src/noise'; import debug from 'debug'; import Libp2p, { Connection, Libp2pModules, Libp2pOptions } from 'libp2p'; -import Bootstrap from 'libp2p-bootstrap'; +import Libp2pBootstrap from 'libp2p-bootstrap'; import { MuxedStream } from 'libp2p-interfaces/dist/src/stream-muxer/types'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: No types available @@ -18,7 +18,8 @@ import Ping from 'libp2p/src/ping'; import { Multiaddr, multiaddr } from 'multiaddr'; import PeerId from 'peer-id'; -import { getBootstrapNodes } from './discovery'; +import { Bootstrap } from './discovery'; +import { BootstrapOptions } from './discovery/bootstrap'; import { getPeersForProtocol } from './select_peer'; import { LightPushCodec, WakuLightPush } from './waku_light_push'; import { WakuMessage } from './waku_message'; @@ -86,15 +87,12 @@ export interface CreateOptions { /** * Use libp2p-bootstrap to discover and connect to new nodes. * - * You can pass: - * - `true` to use {@link getBootstrapNodes}, - * - an array of multiaddresses, - * - a function that returns an array of multiaddresses (or Promise of). + * See [[BootstrapOptions]] for available parameters. * * Note: It overrides any other peerDiscovery modules that may have been set via * {@link CreateOptions.libp2p}. */ - bootstrap?: boolean | string[] | (() => string[] | Promise); + bootstrap?: BootstrapOptions; decryptionKeys?: Array; } @@ -189,29 +187,19 @@ export class Waku { }); if (options?.bootstrap) { - let bootstrap: undefined | (() => string[] | Promise); + const bootstrap = new Bootstrap(options?.bootstrap); - if (options.bootstrap === true) { - bootstrap = getBootstrapNodes; - } else if (Array.isArray(options.bootstrap)) { - bootstrap = (): string[] => { - return options.bootstrap as string[]; - }; - } else if (typeof options.bootstrap === 'function') { - bootstrap = options.bootstrap; - } - - if (bootstrap !== undefined) { + if (bootstrap.getBootstrapPeers !== undefined) { try { - const list = await bootstrap(); + const list = await bootstrap.getBootstrapPeers(); // Note: this overrides any other peer discover libp2pOpts.modules = Object.assign(libp2pOpts.modules, { - peerDiscovery: [Bootstrap], + peerDiscovery: [Libp2pBootstrap], }); libp2pOpts.config.peerDiscovery = { - [Bootstrap.tag]: { + [Libp2pBootstrap.tag]: { list, enabled: true, },