Adds solc option to tests. Runs smart contract tests using remix-tests

Ref: #817

cmd/cmd.js

@@ -278,6 +278,7 @@ class Cmd {
       .option('--nobrowser', __('do not start browser after coverage report is generated'))
       .option('--locale [locale]', __('language to use (default: en)'))
       .option('--loglevel [loglevel]', __('level of logging to display') + ' ["error", "warn", "info", "debug", "trace"]', /^(error|warn|info|debug|trace)$/i, 'warn')
+      .option('--solc', __('run only solidity tests'))
       .description(__('run tests'))
       .action(function(file, options) {
         const node = options.node || 'vm';
@@ -295,7 +296,7 @@ class Cmd {
         }
         checkDeps();
         i18n.setOrDetectLocale(options.locale);
-        embark.runTests({file, loglevel: options.loglevel, gasDetails: options.gasDetails,
+        embark.runTests({file, solc:options.solc, loglevel: options.loglevel, gasDetails: options.gasDetails,
           node: options.node, coverage: options.coverage, noBrowser: options.nobrowser});
       });
   }

lib/tests/run_tests.js

@@ -6,99 +6,78 @@ const fs = require('../core/fs');
 const assert = require('assert');
 const Test = require('./test');
 const EmbarkSpec = require('./reporter');
+const SolcTest = require('./solc_test');
 
 function getFilesFromDir(filePath, cb) {
-  fs.readdir(filePath, (err, files) => {
+  fs.stat(filePath, (err, fileStat) => {
+    const errorMessage = `File "${filePath}" doesn't exist or you don't have permission to it`.red;
     if (err) {
-      return cb(err);
+      return cb(errorMessage);
     }
-    const testFiles = files.filter((file) => {
+    let isDirectory = fileStat.isDirectory();
-      // Only keep the .js files
+    if (isDirectory) {
-      // TODO: make this a configuration in embark.json
+      return fs.readdir(filePath, (err, files) => {
-      return file.substr(-3) === '.js';
+        if (err) {
-    }).map((file) => {
+          return cb(err);
-      return path.join(filePath, file);
+        }
-    });
+        async.map(files, (file, _cb) => {
-    cb(null, testFiles);
+          getFilesFromDir(path.join(filePath, file), _cb);
+        }, (err, arr) => {
+          if (err) {
+            return cb(errorMessage);
+          }
+          cb(null, arr.reduce((a,b) => a.concat(b), []));
+        });
+      });
+    }
+    cb(null, [filePath]);
   });
 }
 
-module.exports = {
+function runJSTests(files, options, cb) {
-  run: function (options) {
+  const loglevel = options.loglevel || 'warn';
-    let failures = 0;
+  async.waterfall([
-    let filePath = options.file;
+    function setupGlobalNamespace(next) {
-    const loglevel = options.loglevel || 'warn';
+      // TODO put default config
-    if (!filePath) {
+      const test = new Test({loglevel, node: options.node});
-      filePath = 'test/';
+      global.embark = test;
-    }
+      global.assert = assert;
+      global.config = test.config.bind(test);
 
-    async.waterfall([
+      let deprecatedWarning = function () {
-      function checkIfDir(next) {
+        console.error(__('%s are not supported anymore', 'EmbarkSpec & deployAll').red);
-        if (filePath.substr(-1) === '/') {
+        console.info(__('You can learn about the new revamped tests here: %s', 'https://embark.status.im/docs/testing.html'.underline));
-          return next(null, null);
+        process.exit();
+      };
+
+      global.deployAll = deprecatedWarning;
+      global.EmbarkSpec = {};
+      global.EmbarkSpec.deployAll = deprecatedWarning;
+
+      // Override require to enable `require('Embark/contracts/contractName');`
+      const Module = require('module');
+      const originalRequire = require('module').prototype.require;
+      Module.prototype.require = function (requireName) {
+        if (requireName.startsWith('Embark')) {
+          return test.require(...arguments);
         }
-        fs.stat(filePath, (err, stats) => {
+        return originalRequire.apply(this, arguments);
-          if (err) {
+      };
-            return next(`File "${filePath}" doesn't exist or you don't have permission to it`.red);
-          }
-          if (stats.isDirectory()) {
-            return next(null, null);
-          }
-          next(null, [filePath]);
-        });
-      },
-      function getFiles(files, next) {
-        if (files) {
-          return next(null, files);
-        }
-        getFilesFromDir(filePath, (err, files) => {
-          if (err) {
-            console.error('Error while reading the directory');
-            return next(err);
-          }
-          next(null, files);
-        });
-      },
-      function setupGlobalNamespace(files, next) {
-        // TODO put default config
-        const test = new Test({loglevel, node: options.node, coverage: options.coverage});
-        global.embark = test;
-        global.assert = assert;
-        global.config = test.config.bind(test);
 
-        let deprecatedWarning = function () {
+      // TODO: this global here might not be necessary at all
-          console.error(__('%s are not supported anymore', 'EmbarkSpec & deployAll').red);
+      global.web3 = global.embark.web3;
-          console.info(__('You can learn about the new revamped tests here: %s', 'https://embark.status.im/docs/testing.html'.underline));
-          process.exit();
-        };
 
-        global.deployAll = deprecatedWarning;
+      global.contract = function (describeName, callback) {
-        global.EmbarkSpec = {};
+        return Mocha.describe(describeName, callback);
-        global.EmbarkSpec.deployAll = deprecatedWarning;
+      };
 
-        // Override require to enable `require('Embark/contracts/contractName');`
+      test.init((err) => {
-        const Module = require('module');
+        next(err, files);
-        const originalRequire = require('module').prototype.require;
+      });
-        Module.prototype.require = function (requireName) {
+    },
-          if (requireName.startsWith('Embark')) {
+    function executeForAllFiles(files, next) {
-            return test.require(...arguments);
+      let fns = files.map((file) => {
-          }
+        return (cb) => {
-          return originalRequire.apply(this, arguments);
-        };
-
-        // TODO: this global here might not be necessary at all
-        global.web3 = global.embark.web3;
-
-        global.contract = function (describeName, callback) {
-          return Mocha.describe(describeName, callback);
-        };
-
-        test.init((err) => {
-          next(err, files);
-        });
-      },
-      function executeForAllFiles(files, next) {
-        async.eachLimit(files, 1, (file, eachCb) => {
           const mocha = new Mocha();
           mocha.reporter(EmbarkSpec, {
             events: global.embark.engine.events,
@@ -107,9 +86,7 @@ module.exports = {
           });
 
           mocha.addFile(file);
-
           mocha.suite.timeout(0);
-
           mocha.suite.beforeAll('Wait for deploy', (done) => {
             if (global.embark.needConfig) {
               global.config({});
@@ -119,16 +96,96 @@ module.exports = {
             });
           });
           mocha.run(function (fails) {
-            failures += fails;
             mocha.suite.removeAllListeners();
             // Mocha prints the error already
-            eachCb();
+            cb(null, fails);
           });
-        }, next);
+        };
+      });
+      async.series(fns, next);
+    }
+  ], (err, runs) => {
+    if(err) {
+      return cb(err);
+    }
+    let failures = runs.reduce((acc, val) => acc + val, 0);
+    fs.remove('.embark/contracts', (_err) => {
+      cb(null, {failures});
+    });
+  });
+}
+
+function runSolidityTests(files, options, cb) {
+  console.log('Running solc tests');
+  const loglevel = options.loglevel || 'warn';
+  let solcTest = new SolcTest({loglevel, node: options.node});
+  global.embark = solcTest;
+  async.waterfall([
+    function initEngine(next) {
+      solcTest.init(next);
+    },
+    function setupTests(next) {
+      solcTest.setupTests(files, next);
+    },
+    function runTests(_reciepts ,cb) {
+      let fns = files.map((file) => {
+        return (cb) => {
+          return solcTest.runTests(file, cb);
+        };
+      });
+      async.series(fns, cb);
+    }
+  ], (err, results) => {
+    if(err) return cb(err);
+    let totalPass = 0;
+    let totalFailures = 0;
+    results.forEach((result) => {
+      result.forEach((r) => {
+        totalPass = totalPass + r.passingNum;
+        totalFailures = totalFailures + r.failureNum;
+      });
+    });
+    cb(null, {failures: totalFailures, pass: totalPass});
+  });
+}
+
+module.exports = {
+  run: function (options) {
+    let filePath = options.file;
+    if (!filePath) {
+      filePath = 'test';
+    }
+    async.waterfall([
+      function getFiles(next) {
+        getFilesFromDir(filePath, next);
       },
-      function runCoverage(next) {
+      function groupFiles(files, next) {
+        let jsFiles = files.filter((filename) => filename.substr(-3) === '.js');
+        let solidityFiles = files.filter((filename) => filename.indexOf('_test.sol') > 0);
+        next(null, {jsFiles, solidityFiles});
+      },
+      function runTests(files, next) {
+        const fns = [];
+        if (!options.solc && files.jsFiles.length > 0) {
+          let fn = (callback) => {
+            runJSTests(files.jsFiles, options, callback);
+          };
+          fns.push(fn);
+        }
+        if(files.solidityFiles.length > 0) {
+          let fn = (callback) => {
+            runSolidityTests(files.solidityFiles, options, callback);
+          };
+          fns.push(fn);
+        }
+        if(fns.length === 0){
+          return next('No tests to run');
+        }
+        async.series(fns, next);
+      },
+      function runCoverage(results, next) {
         if (!options.coverage) {
-          return next();
+          return next(null, results);
         }
 
         global.embark.events.emit('tests:finished', function() {
@@ -139,9 +196,9 @@ module.exports = {
               }
               console.log(`Coverage report created. You can find it here: ${fs.dappPath('coverage/__root__/index.html')}\n`);
               const opn = require('opn');
-              const _next = () => { next(); };
+              const _next = () => { next(null, results); };
               if (options.noBrowser) {
-                return next();
+                return next(null, results);
               }
               opn(fs.dappPath('coverage/__root__/index.html'), {wait: false})
                 .then(() => new Promise(resolve => setTimeout(resolve, 1000)))
@@ -149,21 +206,18 @@ module.exports = {
             });
         });
       }
-    ], (err) => {
+    ], (err, results) => {
       if (err) {
         console.error(err);
         process.exit(1);
       }
-      if (failures) {
+      let totalFailures = results.reduce((acc, result) => acc + result.failures, 0);
-        console.error(` > Total number of failures: ${failures}`.red.bold);
+      if (totalFailures) {
+        console.error(` > Total number of failures: ${totalFailures}`.red.bold);
       } else {
         console.log(' > All tests passed'.green.bold);
       }
-
+      process.exit(totalFailures);
-      // Clean contracts folder for next test run
-      fs.remove('.embark/contracts', (_err) => {
-        process.exit(failures);
-      });
     });
   }
 };

lib/tests/solc_test.js

@@ -0,0 +1,124 @@
+const Test = require('./test');
+const async = require('async');
+const fs = require('fs-extra');
+const File = require('./../core/file');
+const remixTests = require('remix-tests');
+const Base = require('mocha/lib/reporters/base');
+const color = Base.color;
+
+class SolcTest extends Test {
+  constructor(options) {
+    super(options);
+    this.assertLibCode = remixTests.assertLibCode;
+  }
+  init(cb) {
+    const self = this;
+    super.init(() => {
+      let assertFile = new File({
+        filename: 'remix_tests.sol',
+        type: File.types.custom,
+        path: 'remix_tests.sol',
+        resolver: (callback) => {
+          callback(self.assertLibCode);
+        }});
+      self.engine.config.contractsFiles.push(assertFile);
+      cb();
+    });
+  }
+  setupTests(files, cb) {
+    const self = this;
+    files.forEach((file) => {
+      let testFile = self._prepareContractForTest(file);
+      self.engine.config.contractsFiles.push(new File({filename: file, type: File.types.custom, path: file, resolver: function (callback) {
+        callback(testFile);
+      }}));
+    });
+
+    async.waterfall([
+      function initWeb3Provider(next) {
+        self.initWeb3Provider(next);
+      },
+      function resetContracts(next) {
+        self.engine.events.request("contracts:reset:dependencies", next);
+      },
+      function compile(next) {
+        console.info('Compiling contracts'.cyan);
+        self.engine.events.request("contracts:build", false, next);
+      },
+      function determineContractsToDeploy(next) {
+        self.engine.events.request("contracts:list", (err, contracts) => {
+          let contractsToDeploy = contracts.filter((contract) => contract.filename.indexOf('_test.sol') >=0);
+          let assertLib = contracts.filter((contract) => contract.filename === 'remix_tests.sol')[0];
+          next(null, [assertLib].concat(contractsToDeploy));
+        });
+      },
+      function deployContracts(contracts, next) {
+        console.info('Deploying contracts'.cyan);
+        let fns = [];
+        contracts.forEach((contract) => {
+          contract._gasLimit = self.gasLimit;
+          let fn = (cb) => {
+            self.engine.events.request('deploy:contract', contract, cb);
+          };
+          fns.push(fn);
+        });
+        async.series(fns, next);
+      }
+    ],cb);
+  }
+  runTests(file, cb) {
+    console.info('Running tests'.cyan);
+    const self = this;
+    self.engine.events.request('contracts:all', (err, contracts) => {
+      let contractsToTest = [];
+      Object.keys(contracts).forEach((contract) => {
+        if(contracts[contract].filename === file) {
+          contractsToTest.push(contracts[contract]);
+        }
+      });
+      let fns = [];
+      contractsToTest.forEach((contract) => {
+        let contractObject = self._convertToWeb3(contract);
+        let fn = (_callback) => {
+          // TODO: web3 is not injected into the function. Issue has been raised on remixTests.
+          // To fix once web3 has been made injectable.
+          remixTests.runTest(contract.className, contractObject, self._prettyPrint.bind(self), _callback);
+        };
+        fns.push(fn);
+      });
+      async.series(fns, cb);
+    });
+  }
+  _convertToWeb3(contract) {
+    let contractObject = new this.web3.eth.Contract(contract.abiDefinition);
+    contractObject.options.address = contract.deployedAddress;
+    contractObject.options.from = contract.deploymentAccount;
+    contractObject.options.gas = contract.gas;
+    contractObject.filename = contract.filename;
+    return contractObject;
+  }
+  // dynamically insert Assert library as an import
+  // regexIndexOf has been added to String's prototype in remix-tests module
+  _prepareContractForTest(file) {
+    let c = fs.readFileSync(file).toString();
+    const s = /^(import)\s['"](remix_tests.sol)['"];/gm;
+    if (c.regexIndexOf(s) < 0) {
+      c = c.replace(/(pragma solidity \^\d+\.\d+\.\d+;)/, '$1\nimport \"remix_tests.sol\";');
+    }
+    return c;
+  }
+
+  _prettyPrint(obj) {
+    if (obj.type === 'contract') {
+      console.log(color('suite', '%s'), obj.value);
+    } else if(obj.type === 'testPass') {
+      let fmt = color('checkmark', '  ' + Base.symbols.ok) + color('pass', ' %s');
+      console.log(fmt, obj.value);
+    } else if(obj.type === 'testFailure') {
+      let fmt = color('fail', '  %s %s');
+      console.log(fmt, Base.symbols.err, obj.value);
+    }
+  }
+}
+
+module.exports = SolcTest;

lib/tests/test.js

@@ -41,6 +41,12 @@ class Test {
     this.logsSubscription = null;
     this.needConfig = true;
     this.web3 = new Web3();
+    this.engine = new Engine({
+      env: this.options.env || 'test',
+      // TODO: config will need to detect if this is a obj
+      embarkConfig: this.options.embarkConfig || 'embark.json',
+      interceptLogs: false
+    });
   }
 
   initWeb3Provider(callback) {
@@ -150,17 +156,12 @@ class Test {
       compileOnceOnly: true,
       disableOptimizations: true
     });
-    this.events.request('deploy:setGasLimit', 6000000);
+    this.gasLimit = 6000000;
+    this.engine.events.request('deploy:setGasLimit', this.gasLimit);
   }
 
   init(callback) {
     let self = this;
-    this.engine = new Engine({
-      env: this.options.env || 'test',
-      // TODO: config will need to detect if this is a obj
-      embarkConfig: this.options.embarkConfig || 'embark.json',
-      interceptLogs: false
-    });
     async.waterfall([
       function initEngine(cb) {
         self.engine.init({

package.json

@@ -89,6 +89,7 @@
     "pkg-up": "2.0.0",
     "promptly": "2.2.0",
     "propose": "0.0.5",
+    "remix-tests": "0.0.13",
     "request": "2.88.0",
     "sass-loader": "7.1.0",
     "serve-static": "1.13.2",
@@ -106,8 +107,8 @@
     "viz.js": "1.8.2",
     "web3": "1.0.0-beta.34",
     "webpack": "4.19.0",
-    "window-size": "1.1.1",
+    "websocket": "1.0.28",
-    "ws": "^6.0.0"
+    "window-size": "1.1.1"
   },
   "author": "Iuri Matias <iuri.matias@gmail.com>",
   "contributors": [],

test_apps/contracts_app/contracts/simple_storage.sol

@@ -2,14 +2,6 @@ pragma solidity ^0.4.17;
 
 import "ownable.sol";
 
-library Assert {
-  event TestEvent(bool passed, string message);
-
-  function triggerEvent(bool passed, string message) internal {
-    emit TestEvent(passed, message);
-  }
-}
-
 contract SimpleStorage is Ownable {
   uint public storedData;
 
@@ -24,7 +16,6 @@ contract SimpleStorage is Ownable {
     for(uint i = 0; i < 1000; i++) {
       storedData += i;
     }
-    Assert.triggerEvent(true, "hi");
   }
 
   function set2(uint x) public onlyOwner {

test_apps/contracts_app/test/ownable_test.sol

@@ -0,0 +1,17 @@
+pragma solidity ^0.4.7;
+import "./../ownable.sol";
+
+contract OwnableTests {
+  Ownable own;
+
+  function beforeAll() {
+    own = new Ownable();
+  }
+
+  function beforeEach() {
+  }
+
+  function shouldnotbezeroAddress() public {
+    Assert.equal(true, true, "owner is uninitialized");
+  }
+}