Merge pull request #1 from status-im/reorg-tests

This commit is contained in:
Franck Royer 2021-03-22 22:04:55 +11:00 committed by GitHub
commit 5733fda0ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1130 additions and 7599 deletions

View File

@ -3,7 +3,7 @@
"parser": "@typescript-eslint/parser",
"parserOptions": { "project": "./tsconfig.json" },
"env": { "es6": true },
"ignorePatterns": ["node_modules", "build", "coverage", "gen"],
"ignorePatterns": ["node_modules", "build", "coverage", "proto"],
"plugins": ["import", "eslint-comments", "functional"],
"extends": [
"eslint:recommended",

View File

@ -22,6 +22,7 @@ jobs:
run: cd nim-waku && echo "::set-output name=ref::$(git rev-parse HEAD)"
- name: Cache nim-waku binary
id: cache-nim-waku
uses: actions/cache@v2
with:
path: |
@ -29,6 +30,13 @@ jobs:
./nim-waku/vendor/rln/target/debug
key: nim-waku-build-v2-${{ steps.nim-waku-head.outputs.ref }}
# This would have been done part of npm pretest but it gives better
# visibility in the CI if done as a separate step
- name: Build wakunode2
if: steps.cache-nim-waku.outputs.cache-hit != 'true'
shell: bash
run: cd nim-waku && make wakunode2
- name: Install bufbuild
uses: mu-io/setup-buf@v1beta
with:
@ -44,6 +52,12 @@ jobs:
with:
node-version: '14'
- name: Cache npm cache
uses: actions/cache@v2
with:
path: ~/.npm
key: node-v1-${{ hashFiles('**/package-lock.json') }}
- name: install using npm ci
uses: bahmutov/npm-install@v1

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ src/gen
coverage
*.log
yarn.lock
src/proto/**/*.ts

5
.mocharc.json Normal file
View File

@ -0,0 +1,5 @@
{
"extension": ["ts"],
"spec": "src/**/*.spec.ts",
"require": "ts-node/register"
}

View File

@ -1,11 +1,6 @@
version: v1beta1
plugins:
- name: ts
out: src/gen/proto
- name: ts_proto
out: ./src/proto
opt: grpc_js
# protoc 3.13 our above is needed as the schema is v3 with optional fields
- name: js
out: build/main/gen/proto
opt: import_style=commonjs,binary

8167
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -9,10 +9,10 @@
"license": "MIT",
"keywords": [],
"scripts": {
"build": "run-p build:*",
"build": "run-s build:*",
"build:1-proto": "buf generate",
"build:main": "tsc -p tsconfig.json",
"build:module": "tsc -p tsconfig.module.json",
"build:proto": "buf generate",
"fix": "run-s fix:*",
"fix:prettier": "prettier \"src/**/*.ts\" --write",
"fix:lint": "eslint src --ext .ts --fix",
@ -23,13 +23,13 @@
"test:lint": "eslint src --ext .ts",
"test:prettier": "prettier \"src/**/*.ts\" --list-different",
"test:spelling": "cspell \"{README.md,.github/*.md,src/**/*.ts}\"",
"test:unit": "nyc --silent jest",
"test:unit": "nyc --silent mocha --exit # TODO: Remove `--exit` and fix hanging processes",
"test:lint-proto": "buf lint",
"check-cli": "run-s test diff-integration-tests check-integration-tests",
"check-integration-tests": "run-s check-integration-test:*",
"diff-integration-tests": "mkdir -p diff && rm -rf diff/test && cp -r test diff/test && rm -rf diff/test/test-*/.git && cd diff && git init --quiet && git add -A && git commit --quiet --no-verify --allow-empty -m 'WIP' && echo '\\n\\nCommitted most recent integration test output in the \"diff\" directory. Review the changes with \"cd diff && git diff HEAD\" or your preferred git diff viewer.'",
"watch:build": "tsc -p tsconfig.json -w",
"watch:test": "nyc --silent jest --watch",
"watch:test": "nyc --silent mocha --watch",
"cov": "run-s build test:unit cov:html cov:lcov && open-cli coverage/index.html",
"cov:html": "nyc report --reporter=html",
"cov:lcov": "nyc report --reporter=lcov",
@ -57,20 +57,23 @@
"libp2p-tcp": "^0.15.3",
"multiaddr": "^8.1.2",
"prompt-sync": "^4.2.0",
"ts-proto": "^1.74.0",
"yarg": "^1.0.8"
},
"devDependencies": {
"@istanbuljs/nyc-config-typescript": "^1.0.1",
"@types/app-root-path": "^1.2.4",
"@types/axios": "^0.14.0",
"@types/chai": "^4.2.15",
"@types/google-protobuf": "^3.7.4",
"@types/jest": "^26.0.20",
"@types/mocha": "^8.2.2",
"@types/node": "^14.14.31",
"@types/tail": "^2.0.0",
"@typescript-eslint/eslint-plugin": "^4.0.1",
"@typescript-eslint/parser": "^4.0.1",
"app-root-path": "^3.0.0",
"axios": "^0.21.1",
"chai": "^4.3.4",
"codecov": "^3.5.0",
"cspell": "^4.1.0",
"cz-conventional-changelog": "^3.3.0",
@ -81,9 +84,7 @@
"eslint-plugin-import": "^2.22.0",
"fast-check": "^2.14.0",
"gh-pages": "^3.1.0",
"grpc_tools_node_protoc_ts": "^5.1.3",
"jest": "^26.6.3",
"jest-fast-check": "^1.0.2",
"mocha": "^8.3.2",
"npm-run-all": "^4.1.5",
"nyc": "^15.1.0",
"open-cli": "^6.0.1",
@ -91,7 +92,7 @@
"prettier": "^2.1.1",
"standard-version": "^9.0.0",
"tail": "^2.2.0",
"ts-node": "^9.0.0",
"ts-node": "^9.1.1",
"typedoc": "^0.20.29",
"typescript": "^4.0.2"
},

View File

@ -1,59 +1,14 @@
import 'jest';
import Libp2p from 'libp2p';
import Pubsub from 'libp2p-interfaces/src/pubsub';
import { expect } from 'chai';
import { NimWaku } from '../test_utils/nim_waku';
import Waku from './waku';
import { Message } from './waku_message';
import { CODEC, TOPIC } from './waku_relay';
import { CODEC } from './waku_relay';
// TODO: Fix this, see https://github.com/ChainSafe/js-libp2p-gossipsub/issues/151
test.skip('Publishes message', async () => {
const message = Message.fromUtf8String('Bird bird bird, bird is the word!');
const [waku1, waku2] = await Promise.all([Waku.create(), Waku.create()]);
// Add node's 2 data to the PeerStore
waku1.libp2p.peerStore.addressBook.set(
waku2.libp2p.peerId,
waku2.libp2p.multiaddrs
);
await waku1.libp2p.dial(waku2.libp2p.peerId);
await waku2.relay.subscribe();
await new Promise((resolve) =>
waku2.libp2p.pubsub.once('pubsub:subscription-change', (...args) =>
resolve(args)
)
);
// Setup the promise before publishing to ensure the event is not missed
const promise = waitForNextData(waku1.libp2p.pubsub);
await waku2.relay.publish(message);
const node1Received = await promise;
expect(node1Received.isEqualTo(message)).toBeTruthy();
});
test('Registers waku relay protocol', async () => {
const waku = await Waku.create();
const protocols = Array.from(waku.libp2p.upgrader.protocols.keys());
expect(protocols.findIndex((value) => value == CODEC)).toBeTruthy();
});
test('Does not register any sub protocol', async () => {
const waku = await Waku.create();
const protocols = Array.from(waku.libp2p.upgrader.protocols.keys());
expect(protocols.findIndex((value) => value.match(/sub/))).toBeTruthy();
});
test('Nim-interop: nim-waku node connects to js node', async () => {
describe('Waku', function () {
describe('Interop: Nim', function () {
it('nim connects to js', async function () {
this.timeout(10_000);
const waku = await Waku.create();
const peerId = waku.libp2p.peerId.toB58String();
@ -63,12 +18,12 @@ test('Nim-interop: nim-waku node connects to js node', async () => {
);
const multiAddrWithId = localMultiaddr + '/p2p/' + peerId;
const nimWaku = new NimWaku(expect.getState().currentTestName);
const nimWaku = new NimWaku(this.test!.title);
await nimWaku.start({ staticnode: multiAddrWithId });
const nimPeers = await nimWaku.peers();
expect(nimPeers).toEqual([
expect(nimPeers).to.deep.equal([
{
multiaddr: multiAddrWithId,
protocol: CODEC,
@ -79,115 +34,10 @@ test('Nim-interop: nim-waku node connects to js node', async () => {
const nimPeerId = await nimWaku.getPeerId();
const jsPeers = waku.libp2p.peerStore.peers;
expect(jsPeers.has(nimPeerId.toB58String())).toBeTruthy();
expect(jsPeers.has(nimPeerId.toB58String())).to.be.true;
nimWaku.stop();
await waku.stop();
});
test('Nim-interop: js node receives default subscription from nim node', async () => {
const waku = await Waku.create();
const peerId = waku.libp2p.peerId.toB58String();
const localMultiaddr = waku.libp2p.multiaddrs.find((addr) =>
addr.toString().match(/127\.0\.0\.1/)
);
const multiAddrWithId = localMultiaddr + '/p2p/' + peerId;
const nimWaku = new NimWaku(expect.getState().currentTestName);
await nimWaku.start({ staticnode: multiAddrWithId });
const nimPeerId = await nimWaku.getPeerId();
const subscribers = waku.libp2p.pubsub.getSubscribers(TOPIC);
expect(subscribers).toContain(nimPeerId.toB58String());
});
test('Nim-interop: js node sends message to nim node', async () => {
const message = Message.fromUtf8String('This is a message');
const waku = await Waku.create();
// TODO: nim-waku does follow the `StrictNoSign` policy hence we need to change
// it for nim-waku to process our messages. Can be removed once
// https://github.com/status-im/nim-waku/issues/422 is fixed
waku.libp2p.pubsub.globalSignaturePolicy = 'StrictSign';
const peerId = waku.libp2p.peerId.toB58String();
const localMultiaddr = waku.libp2p.multiaddrs.find((addr) =>
addr.toString().match(/127\.0\.0\.1/)
);
const multiAddrWithId = localMultiaddr + '/p2p/' + peerId;
const nimWaku = new NimWaku(expect.getState().currentTestName);
await nimWaku.start({ staticnode: multiAddrWithId });
await patchPeerStore(nimWaku, waku.libp2p);
await waku.relay.publish(message);
await nimWaku.waitForLog('WakuMessage received');
const msgs = await nimWaku.messages();
expect(msgs[0].contentTopic).toEqual(message.contentTopic);
expect(msgs[0].version).toEqual(message.version);
const payload = Buffer.from(msgs[0].payload);
expect(Buffer.compare(payload, message.payload)).toBe(0);
});
test('Nim-interop: nim node sends message to js node', async () => {
const message = Message.fromUtf8String('Here is another message.');
const waku = await Waku.create();
const peerId = waku.libp2p.peerId.toB58String();
const localMultiaddr = waku.libp2p.multiaddrs.find((addr) =>
addr.toString().match(/127\.0\.0\.1/)
);
const multiAddrWithId = localMultiaddr + '/p2p/' + peerId;
const nimWaku = new NimWaku(expect.getState().currentTestName);
await nimWaku.start({ staticnode: multiAddrWithId });
await patchPeerStore(nimWaku, waku.libp2p);
await waku.relay.subscribe();
await new Promise((resolve) =>
waku.libp2p.pubsub.once('gossipsub:heartbeat', resolve)
);
const receivedPromise = waitForNextData(waku.libp2p.pubsub);
await nimWaku.sendMessage(message);
const receivedMsg = await receivedPromise;
expect(receivedMsg.contentTopic).toBe(message.contentTopic);
expect(receivedMsg.version).toBe(message.version);
const payload = Buffer.from(receivedMsg.payload);
expect(Buffer.compare(payload, message.payload)).toBe(0);
});
function waitForNextData(pubsub: Pubsub): Promise<Message> {
return new Promise((resolve) => {
pubsub.once(TOPIC, resolve);
}).then((msg: any) => {
return Message.fromBinary(msg.data);
});
}
// TODO: Remove this hack, tracked with https://github.com/status-im/nim-waku/issues/419
async function patchPeerStore(nimWaku: NimWaku, node: Libp2p) {
const nimPeerId = await nimWaku.getPeerId();
node.identifyService!.peerStore.protoBook.set(nimPeerId, [CODEC]);
const peer = node.peerStore.peers.get(nimPeerId.toB58String());
if (!peer) {
throw 'Did not find nim-waku node in peers';
}
peer.protocols = [CODEC];
node.peerStore.peers.set(nimPeerId.toB58String(), peer);
await new Promise((resolve) =>
node.pubsub.once('gossipsub:heartbeat', resolve)
);
}

View File

@ -27,4 +27,8 @@ export default class Waku {
return new Waku(libp2p, new WakuRelay(libp2p.pubsub));
}
async stop() {
await this.libp2p.stop();
}
}

View File

@ -1,15 +1,17 @@
import { fc, testProp } from 'jest-fast-check';
import fc from 'fast-check';
import { Message } from './waku_message';
testProp(
'Waku message round trip binary serialization',
[fc.fullUnicodeString()],
(s) => {
describe('Waku Message', function () {
it('Waku message round trip binary serialization', function () {
fc.assert(
fc.property(fc.string(), (s) => {
const msg = Message.fromUtf8String(s);
const binary = msg.toBinary();
const actual = Message.fromBinary(binary);
expect(actual.isEqualTo(msg)).toBeTruthy();
}
return actual.isEqualTo(msg);
})
);
});
});

View File

@ -1,28 +1,18 @@
import { WakuMessage } from '../gen/proto/waku/v2/waku_pb';
// Ensure that this class matches the proto interface while
import { Reader } from 'protobufjs/minimal';
// Protecting the user from protobuf oddities
import { WakuMessage } from '../proto/waku/v2/waku';
const DEFAULT_CONTENT_TOPIC = 1;
const DEFAULT_VERSION = 0;
export class Message {
public payload: Uint8Array;
public contentTopic: number;
public version: number;
private constructor(public protobuf: WakuMessage) {
this.protobuf = protobuf;
const msg = protobuf.toObject();
// Let's make is easier to avoid mistakes and only store in Uint8Array format
let payload;
if (typeof msg.payload === 'string') {
payload = Buffer.from(msg.payload, 'base64');
} else {
payload = msg.payload;
}
this.payload = payload;
this.contentTopic = msg.contentTopic;
this.version = msg.version;
}
private constructor(
public payload?: Uint8Array,
public contentTopic?: number,
public version?: number
) {}
/**
* Create Message from utf-8 string
@ -30,39 +20,33 @@ export class Message {
* @returns {Message}
*/
static fromUtf8String(message: string): Message {
const wakuMsg = new WakuMessage();
// Only Version 0 is implemented in Waku 2.
// 0: payload SHOULD be either plain or that encryption is done at a separate layer outside of Waku.
wakuMsg.setVersion(0);
// This is the content topic commonly used at this time
wakuMsg.setContentTopic(1);
const buf = Buffer.from(message, 'utf-8');
// Only accepts Uint8Array or base64 string
wakuMsg.setPayload(buf);
return new Message(wakuMsg);
const payload = Buffer.from(message, 'utf-8');
return new Message(payload, DEFAULT_CONTENT_TOPIC, DEFAULT_VERSION);
}
static fromBinary(message: Uint8Array): Message {
const wakuMsg = WakuMessage.deserializeBinary(message);
return new Message(wakuMsg);
static fromBinary(bytes: Uint8Array): Message {
const wakuMsg = WakuMessage.decode(Reader.create(bytes));
return new Message(wakuMsg.payload, wakuMsg.contentTopic, wakuMsg.version);
}
toBinary(): Uint8Array {
return this.protobuf.serializeBinary();
return WakuMessage.encode({
payload: this.payload,
version: this.version,
contentTopic: this.contentTopic,
}).finish();
}
// Purely for tests purposes.
// We do consider protobuf field when checking equality
// As the content is held by the other fields.
// TODO: Consider using WakuMessage.equals
isEqualTo(other: Message) {
const payloadsAreEqual =
this.payload && other.payload
? Buffer.compare(this.payload, other.payload) === 0
: !(this.payload || other.payload);
return (
Buffer.compare(this.payload, other.payload) === 0 &&
payloadsAreEqual &&
this.contentTopic === other.contentTopic &&
this.version === other.version
);

166
src/lib/waku_relay.spec.ts Normal file
View File

@ -0,0 +1,166 @@
import { expect } from 'chai';
import Libp2p from 'libp2p';
import Pubsub from 'libp2p-interfaces/src/pubsub';
import { NimWaku } from '../test_utils/nim_waku';
import Waku from './waku';
import { Message } from './waku_message';
import { CODEC, TOPIC } from './waku_relay';
describe('Waku Relay', () => {
// TODO: Fix this, see https://github.com/ChainSafe/js-libp2p-gossipsub/issues/151
it.skip('Publish', async () => {
const message = Message.fromUtf8String('Bird bird bird, bird is the word!');
const [waku1, waku2] = await Promise.all([Waku.create(), Waku.create()]);
// Add node's 2 data to the PeerStore
waku1.libp2p.peerStore.addressBook.set(
waku2.libp2p.peerId,
waku2.libp2p.multiaddrs
);
await waku1.libp2p.dial(waku2.libp2p.peerId);
await waku2.relay.subscribe();
await new Promise((resolve) =>
waku2.libp2p.pubsub.once('pubsub:subscription-change', (...args) =>
resolve(args)
)
);
// Setup the promise before publishing to ensure the event is not missed
const promise = waitForNextData(waku1.libp2p.pubsub);
await waku2.relay.publish(message);
const node1Received = await promise;
expect(node1Received.isEqualTo(message)).to.be.true;
await Promise.all([waku1.stop(), waku2.stop()]);
});
it('Registers waku relay protocol', async function () {
const waku = await Waku.create();
const protocols = Array.from(waku.libp2p.upgrader.protocols.keys());
expect(protocols).to.contain(CODEC);
await waku.stop();
});
it('Does not register any sub protocol', async function () {
const waku = await Waku.create();
const protocols = Array.from(waku.libp2p.upgrader.protocols.keys());
expect(protocols.findIndex((value) => value.match(/sub/))).to.eq(-1);
await waku.stop();
});
describe('Interop: Nim', function () {
let waku: Waku;
let nimWaku: NimWaku;
beforeEach(async function () {
this.timeout(10_000);
waku = await Waku.create();
const peerId = waku.libp2p.peerId.toB58String();
const localMultiaddr = waku.libp2p.multiaddrs.find((addr) =>
addr.toString().match(/127\.0\.0\.1/)
);
const multiAddrWithId = localMultiaddr + '/p2p/' + peerId;
nimWaku = new NimWaku(this.test!.title);
await nimWaku.start({ staticnode: multiAddrWithId });
});
afterEach(async function () {
nimWaku ? nimWaku.stop() : null;
waku ? await waku.stop() : null;
});
it('nim subscribes to js', async function () {
const nimPeerId = await nimWaku.getPeerId();
const subscribers = waku.libp2p.pubsub.getSubscribers(TOPIC);
expect(subscribers).to.contain(nimPeerId.toB58String());
});
it('Js publishes to nim', async function () {
this.timeout(5000);
const message = Message.fromUtf8String('This is a message');
// TODO: nim-waku does follow the `StrictNoSign` policy hence we need to change
// it for nim-waku to process our messages. Can be removed once
// https://github.com/status-im/nim-waku/issues/422 is fixed
waku.libp2p.pubsub.globalSignaturePolicy = 'StrictSign';
await patchPeerStore(nimWaku, waku.libp2p);
await waku.relay.publish(message);
await nimWaku.waitForLog('WakuMessage received');
const msgs = await nimWaku.messages();
expect(msgs[0].contentTopic).to.equal(message.contentTopic);
expect(msgs[0].version).to.equal(message.version);
const payload = Buffer.from(msgs[0].payload);
expect(Buffer.compare(payload, message.payload!)).to.equal(0);
});
it('Nim publishes to js', async function () {
this.timeout(5000);
const message = Message.fromUtf8String('Here is another message.');
await patchPeerStore(nimWaku, waku.libp2p);
await waku.relay.subscribe();
await new Promise((resolve) =>
waku.libp2p.pubsub.once('gossipsub:heartbeat', resolve)
);
const receivedPromise = waitForNextData(waku.libp2p.pubsub);
await nimWaku.sendMessage(message);
const receivedMsg = await receivedPromise;
expect(receivedMsg.contentTopic).to.eq(message.contentTopic);
expect(receivedMsg.version).to.eq(message.version);
const payload = Buffer.from(receivedMsg.payload!);
expect(Buffer.compare(payload, message.payload!)).to.eq(0);
});
});
});
function waitForNextData(pubsub: Pubsub): Promise<Message> {
return new Promise((resolve) => {
pubsub.once(TOPIC, resolve);
}).then((msg: any) => {
return Message.fromBinary(msg.data);
});
}
// TODO: Remove this hack, tracked with https://github.com/status-im/nim-waku/issues/419
async function patchPeerStore(nimWaku: NimWaku, node: Libp2p) {
const nimPeerId = await nimWaku.getPeerId();
node.identifyService!.peerStore.protoBook.set(nimPeerId, [CODEC]);
const peer = node.peerStore.peers.get(nimPeerId.toB58String());
if (!peer) {
throw 'Did not find nim-waku node in peers';
}
peer.protocols = [CODEC];
node.peerStore.peers.set(nimPeerId.toB58String(), peer);
await new Promise((resolve) =>
node.pubsub.once('gossipsub:heartbeat', resolve)
);
}

View File

@ -1,6 +1,8 @@
import { expect } from 'chai';
import { argsToArray, bufToHex, defaultArgs, strToHex } from './nim_waku';
test('Correctly serialized arguments', () => {
it('Correctly serialized arguments', function () {
const args = defaultArgs();
Object.assign(args, { portsShift: 42 });
@ -15,18 +17,18 @@ test('Correctly serialized arguments', () => {
'--ports-shift=42',
];
expect(actual).toEqual(expected);
expect(actual).to.deep.equal(expected);
});
test('Convert utf-8 string to hex', () => {
it('Convert utf-8 string to hex', function () {
const str = 'This is an utf-8 string.';
const expected = '0x5468697320697320616e207574662d3820737472696e672e';
const actual = strToHex(str);
expect(actual).toEqual(expected);
expect(actual).deep.equal(expected);
});
test('Convert buffer to hex', () => {
it('Convert buffer to hex', function () {
const buf = Uint8Array.from([
0x54,
0x68,
@ -56,5 +58,5 @@ test('Convert buffer to hex', () => {
const expected = '0x5468697320697320616e207574662d3820737472696e672e';
const actual = bufToHex(buf);
expect(actual).toEqual(expected);
expect(actual).to.deep.equal(expected);
});

View File

@ -74,13 +74,14 @@ export class NimWaku {
],
});
this.process.on('exit', (signal) => {
console.log(`ERROR: nim-waku node stopped: ${signal}`);
});
await this.waitForLog('RPC Server started');
}
public stop() {
this.process ? this.process.kill('SIGINT') : null;
this.process = undefined;
}
async waitForLog(msg: string) {
return waitForLine(this.logPath, msg);
}
@ -108,6 +109,10 @@ export class NimWaku {
async sendMessage(message: Message) {
this.checkProcess();
if (!message.payload) {
throw 'Attempting to send empty message';
}
const rpcMessage = {
payload: bufToHex(message.payload),
contentTopic: message.contentTopic,

View File

@ -41,10 +41,13 @@
// "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
"lib": ["es2017"],
"types": ["node", "jest"],
"typeRoots": ["node_modules/@types", "src/types"]
"types": ["node", "mocha"],
"typeRoots": ["node_modules/@types", "src/types", "src/gen/proto"]
},
"include": ["src/**/*.ts"],
"include": ["src/**/*.ts", "src/**/*.d.ts"],
"exclude": ["node_modules/**"],
"compileOnSave": false
"compileOnSave": false,
"ts-node": {
"files": true
}
}