chore: test runner tests (#1884)

* chore: test runner tests

* chore: mocha test runner tests

* chore: add solidity test file to dapp

dapps/tests/app/test/solidity_test.sol

@@ -0,0 +1,14 @@
+pragma solidity ^0.4.25;
+import 'remix_tests.sol';
+
+contract MyTest {
+  uint i = 0;
+
+  function beforeAll() {
+    i += 1;
+  }
+
+  function valueShouldBe1() public {
+    Assert.equal(1, i, "value is not correct");
+  }
+}

packages/plugins/mocha-tests/package.json

@@ -51,7 +51,9 @@
     "cross-env": "5.2.0",
     "eslint": "5.7.0",
     "npm-run-all": "4.1.5",
+    "refute": "1.0.2",
     "rimraf": "3.0.0",
+    "sinon": "7.4.2",
     "tslint": "5.16.0",
     "typescript": "3.4.5"
   },

packages/plugins/mocha-tests/src/test/mocha_tests_test.js

@@ -1,8 +1,47 @@
 /* globals describe, it */
 const assert = require('assert').strict;
+const refute = require('refute')(assert);
+const sinon = require('sinon');
+
+const MochaTestRunner = require('../lib');
+
+const validFiles = [
+  'SOME_UPPERCASE_TEST.js',
+  'other_uppercase_test.JS',
+  'some_test_file.js',
+  'c:\test\file.js'
+];
+
+const invalidFiles = [
+  'jsschool.com',
+  'other_uppercase_test.js.map'
+];
 
 describe('Mocha Test Runner', () => {
-  it('should have tests', (_done) => {
-    assert(false, 'No tests yet on embark-mocha-tests');
+  let embark;
+
+  beforeEach(() => {
+    const events = { request: sinon.spy() };
+    embark = { events: events };
+  });
+
+  describe('methods', () => {
+    let instance;
+
+    beforeEach(() => { instance = new MochaTestRunner(embark, {}); });
+
+    describe('match', () => {
+      it('matches .js files', () => {
+        validFiles.forEach(f => {
+          assert(instance.match(f), `didn't match ${f} when it should`);
+        });
+      });
+
+      it('does not match non .js files', () => {
+        invalidFiles.forEach(f => {
+          refute(instance.match(f), `matched ${f} when it shouldn't`);
+        });
+      });
+    });
   });
 });

packages/plugins/solidity-tests/src/lib/index.js

@@ -1,6 +1,5 @@
 const async = require("async");
 const { dappPath } = require("embark-utils");
-const fs = require("fs");
 const remixTests = require('remix-tests');
 const Web3 = require('web3');
 
@@ -31,6 +30,8 @@ class SolidityTestRunner {
   constructor(embark, options) {
     this.embark = embark;
     this.events = embark.events;
+    this.fs = embark.fs;
+
     this.plugins = options.plugins;
 
     this.files = [];
@@ -57,7 +58,7 @@ class SolidityTestRunner {
 
   run(options, cb) {
     const reporter = new Reporter(options.reporter);
-    const {events, _plugins} = this;
+    const {events, fs} = this;
 
     if (this.files.length === 0) {
       return cb(null, 0);

packages/stack/test-runner/package.json

@@ -24,7 +24,7 @@
     "type": "git",
     "url": "https://github.com/embark-framework/embark.git"
   },
-  "main": "./dist/index.js",
+  "main": "./dist/lib/index.js",
   "scripts": {
     "build": "cross-env BABEL_ENV=node babel src --extensions \".js\" --out-dir dist --root-mode upward --source-maps",
     "ci": "npm run qa",
@@ -40,7 +40,8 @@
     "// typecheck": "tsc",
     "watch": "run-p watch:*",
     "watch:build": "npm run build -- --verbose --watch",
-    "// watch:typecheck": "npm run typecheck -- --preserveWatchOutput --watch"
+    "// watch:typecheck": "npm run typecheck -- --preserveWatchOutput --watch",
+    "test": "mocha dist/test/**/*.js"
   },
   "eslintConfig": {
     "extends": "../../../.eslintrc.json"
@@ -53,7 +54,6 @@
     "embark-utils": "^4.1.1",
     "fs-extra": "7.0.1",
     "istanbul": "0.4.5",
-    "mocha": "6.2.0",
     "open": "6.4.0"
   },
   "devDependencies": {
@@ -62,8 +62,11 @@
     "@types/async": "2.0.50",
     "cross-env": "5.2.0",
     "eslint": "5.7.0",
+    "mocha": "6.2.0",
     "npm-run-all": "4.1.5",
+    "refute": "1.0.2",
     "rimraf": "3.0.0",
+    "sinon": "7.4.2",
     "tslint": "5.16.0",
     "typescript": "3.4.5"
   },

packages/stack/test-runner/src/lib/index.js

@@ -3,7 +3,6 @@ const async = require('async');
 const chalk = require('chalk');
 const path = require('path');
 const { embarkPath, dappPath, runCmd } = require('embark-utils');
-import fs from 'fs';
 import { COVERAGE_GAS_LIMIT, GAS_LIMIT } from './constants';
 
 const Reporter = require('./reporter');
@@ -23,11 +22,11 @@ class TestRunner {
       this.run(options, callback);
     });
 
-    this.events.setCommandHandler('tests:runner:register', (name, matchFn, addFn, runFn) => {
+    this.events.setCommandHandler('tests:runner:register', (pluginName, matchFn, addFn, runFn) => {
       // We unshift to give priority to runners registered after the default ones, making it
       // possible to override the ones Embark ships with. This will open the door for things
       // like Jest tests and such.
-      this.runners.unshift({name, matchFn, addFn, runFn});
+      this.runners.unshift({pluginName, matchFn, addFn, runFn});
     });
   }
 
@@ -92,6 +91,8 @@ class TestRunner {
   }
 
   getFilesFromDir(filePath, cb) {
+    const {fs} = this;
+
     fs.stat(filePath, (err, fileStat) => {
       const errorMessage = `File "${filePath}" doesn't exist or you don't have permission to it`.red;
       if (err) {

packages/stack/test-runner/src/test/test_runner_test.js

@@ -0,0 +1,135 @@
+/* globals describe, it */
+const assert = require('assert').strict;
+const refute = require('refute')(assert);
+const sinon = require('sinon');
+
+const TestRunner = require('../lib/index.js');
+
+describe('Test Runner', () => {
+  let embark;
+  let instance;
+
+  beforeEach(() => {
+    const events = { setCommandHandler: () => {}, on: () => {} };
+    const logger = { warn: sinon.fake() };
+
+    embark = { events, logger };
+    instance = new TestRunner(embark, {});
+  });
+
+  describe('command handlers', () => {
+    describe('tests:run', () => {
+      let first, second;
+
+      beforeEach(() => {
+        first = {
+          pluginName: 'first',
+          matchFn: sinon.fake(path => path === 'test/file_first.js'),
+          addFn: sinon.fake(),
+          runFn: sinon.fake.yields()
+        };
+
+        second = {
+          pluginName: 'second',
+          matchFn: sinon.fake(path => path === 'test/file_second.js'),
+          addFn: sinon.fake(),
+          runFn: sinon.fake.yields()
+        };
+
+        instance.runners = [first, second];
+        instance.getFilesFromDir = (_, cb) => {
+          cb(null, ['test/file_first.js', 'test/file_second.js']);
+        }
+      });
+
+      it('should warn when a file does not match any plugins', (done) => {
+        instance.getFilesFromDir = (_, cb) => {
+          cb(null, ['luri.js']);
+        }
+
+        instance.run({}, (err) => {
+          // Ensure no error was returned
+          refute(err);
+
+          sinon.assert.calledWith(first.matchFn, 'luri.js');
+          sinon.assert.calledWith(second.matchFn, 'luri.js');
+
+          // Ensure that we logged
+          sinon.assert.calledWithMatch(embark.logger.warn, /luri.js/);
+
+          done();
+        });
+      });
+
+      it('should add the correct files to the correct plugins', (done) => {
+        instance.run({}, (err) => {
+          // Ensure no error was returned
+          refute(err);
+
+          // Ensure that we didn't warn that runners weren't registered.
+          sinon.assert.notCalled(embark.logger.warn);
+
+          // Ensure plugins received the correct files
+          sinon.assert.calledWith(first.addFn, 'test/file_first.js');
+          sinon.assert.neverCalledWith(second.addFn, 'test/file_first.js');
+          sinon.assert.calledWith(second.addFn, 'test/file_second.js');
+          sinon.assert.neverCalledWith(first.addFn, 'test/file_second.js');
+
+          done();
+        });
+      });
+
+      it('should run registered plugins in order', (done) => {
+        instance.run({}, (err) => {
+          // Ensure no error was returned
+          refute(err);
+
+          // Ensure plugins are called in order
+          sinon.assert.callOrder(first.runFn, second.runFn);
+
+          done();
+        });
+      });
+
+      it('should not keep going if a plugin raises an error', (done) => {
+        first.runFn = sinon.fake.yields('some error');
+
+        instance.run({}, (err) => {
+          // Ensure an error was returned
+          assert.equal('some error', err);
+
+          // Ensure only first plugin got called
+          sinon.assert.calledOnce(first.runFn);
+          sinon.assert.notCalled(second.runFn);
+
+          done();
+        });
+      });
+
+      it('callback receives number of passes and failures', (done) => {
+        first.runFn = sinon.spy(({reporter}, cb) => {
+          reporter.report('first test #1', 0.01, true);
+          reporter.report('first test #2', 0.01, true);
+          cb();
+        });
+
+        second.runFn = sinon.spy(({reporter}, cb) => {
+          reporter.report('second test #1', 0.01, true);
+          reporter.report('second test #2', 0.01, false, 'failed assertion');
+          cb();
+        });
+
+        instance.run({}, (err, passes, failures) => {
+          // Ensure no error was returned
+          refute(err);
+
+          // Ensure we get the correct reported passes and fails
+          assert.equal(3, passes);
+          assert.equal(1, failures);
+
+          done();
+        });
+      });
+    });
+  });
+});

yarn.lock

@@ -2212,7 +2212,7 @@
     "@sinonjs/commons" "^1"
     "@sinonjs/samsam" "^3.1.0"
 
-"@sinonjs/samsam@^3.1.0", "@sinonjs/samsam@^3.3.2":
+"@sinonjs/samsam@^3.1.0", "@sinonjs/samsam@^3.3.2", "@sinonjs/samsam@^3.3.3":
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-3.3.3.tgz#46682efd9967b259b81136b9f120fd54585feb4a"
   integrity sha512-bKCMKZvWIjYD0BLGnNrxVuw4dkWCYsLqFOUWw8VgKF/+5Y+mE7LfHWPIYoDXowH+3a9LsWDMo0uAP8YDosPvHQ==
@@ -14027,7 +14027,7 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
-nise@^1.2.0, nise@^1.5.1:
+nise@^1.2.0, nise@^1.5.1, nise@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/nise/-/nise-1.5.2.tgz#b6d29af10e48b321b307e10e065199338eeb2652"
   integrity sha512-/6RhOUlicRCbE9s+94qCUsyE+pKlVJ5AhIv+jEE7ESKwnbXqulKZ1FYU+XAtHHWE9TinYvAxDUJAb912PwPoWA==
@@ -18805,6 +18805,19 @@ sinon@7.4.1:
     nise "^1.5.1"
     supports-color "^5.5.0"
 
+sinon@7.4.2:
+  version "7.4.2"
+  resolved "https://registry.yarnpkg.com/sinon/-/sinon-7.4.2.tgz#ecd54158fef2fcfbdb231a3fa55140e8cb02ad6c"
+  integrity sha512-pY5RY99DKelU3pjNxcWo6XqeB1S118GBcVIIdDi6V+h6hevn1izcg2xv1hTHW/sViRXU7sUOxt4wTUJ3gsW2CQ==
+  dependencies:
+    "@sinonjs/commons" "^1.4.0"
+    "@sinonjs/formatio" "^3.2.1"
+    "@sinonjs/samsam" "^3.3.3"
+    diff "^3.5.0"
+    lolex "^4.2.0"
+    nise "^1.5.2"
+    supports-color "^5.5.0"
+
 sisteransi@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-0.1.1.tgz#5431447d5f7d1675aac667ccd0b865a4994cb3ce"
@@ -21813,7 +21826,6 @@ websocket-extensions@>=0.1.1:
   dependencies:
     debug "^2.2.0"
     es5-ext "^0.10.50"
-    gulp "^4.0.2"
     nan "^2.14.0"
     typedarray-to-buffer "^3.1.5"
     yaeti "^0.0.6"