Merge pull request #403 from embark-framework/features/file-per-process

Process files in multiple separate processes

lib/core/engine.js

@@ -8,10 +8,8 @@ const CodeGenerator = require('../contracts/code_generator.js');
 const ServicesMonitor = require('./services_monitor.js');
 const Watch = require('../pipeline/watch.js');
 const LibraryManager = require('../versions/library_manager.js');
-const utils = require('../utils/utils');
+const Pipeline = require('../pipeline/pipeline.js');
-const constants = require('../constants');
 const async = require('async');
-const fs = require('../core/fs.js');
 
 class Engine {
   constructor(options) {
@@ -134,81 +132,34 @@ class Engine {
   }
 
   pipelineService(_options) {
-    let self = this;
+    const self = this;
-    self.events.emit("status", "Building Assets");
+    this.events.emit("status", "Building Assets");
-    const pipelineProcess = require('child_process').fork(utils.joinPath(__dirname, '../pipeline/pipeline.js'));
+    const pipeline = new Pipeline({
+      buildDir: this.config.buildDir,
+      contractsFiles: this.config.contractsFiles,
+      assetFiles: this.config.assetFiles,
+      events: this.events,
+      logger: this.logger,
+      normalizeInput: this.normalizeInput,
+      plugins: this.plugins
+    });
 
-    async.waterfall([
+    const queue = async.queue((task, callback) => {
-      function getWeb3Version(next) {
+      pipeline.build(task.abi, task.contractsJSON, task.path, callback);
-        self.events.request("version:get:web3", function(web3Version) {
+    }, 1);
-          next(null, web3Version);
-        });
-      },
-      function getWeb3Location(web3Version, next) {
-        if (web3Version === "1.0.0-beta") {
-          return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
-        } else {
-          self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
-            return next(err, fs.dappPath(location));
-          });
-        }
-      },
-      function getProviderCode(web3Location, next) {
-        self.events.once('code-generator-ready', function () {
-          self.events.request('provider-code', function(providerCode) {
-            next(null, web3Location, providerCode);
-          });
-        });
-      }
-    ], (err, web3Location, providerCode) => {
-      if (err) {
-        return self.logger.error(err);
-      }
 
-      // Setup Pipeline
+    this.events.on('code-generator-ready', function () {
-      pipelineProcess.send({action: constants.pipeline.init, options: {
+      self.events.request('code', function (abi, contractsJSON) {
-          buildDir: self.config.buildDir,
+        self.currentAbi = abi;
-          contractsFiles: self.config.contractsFiles,
+        self.contractsJSON = contractsJSON;
-          assetFiles: self.config.assetFiles,
-          events: self.events,
-          pipelinePlugins: self.plugins.getPluginsFor('pipeline'),
-          pluginImports: self.plugins.getPluginsProperty('imports', 'imports'),
-          web3Location,
-          providerCode
-        }});
 
-      pipelineProcess.on('message', function (msg) {
+        queue.push({abi, contractsJSON, path: null}, () => {
-        if (msg.result === constants.pipeline.built) {
           if (self.watch) {
             self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched
           }
-          if (msg.error) {
+          self.events.emit('outputDone');
-            return self.logger.error(msg.error);
-          }
-          return self.events.emit('outputDone');
-        }
-
-        if (msg.result === constants.pipeline.log) {
-          if (self.logger[msg.type]) {
-            return self.logger[msg.type](self.normalizeInput(msg.message));
-          }
-          self.logger.debug(self.normalizeInput(msg.message));
-        }
-      });
-
-      function build() {
-        self.events.request('code', function (abi, contractsJSON) {
-          self.currentAbi = abi;
-          self.contractsJSON = contractsJSON;
-          pipelineProcess.send({action: constants.pipeline.build, abi, contractsJSON, path: null});
         });
-      }
-
-      self.events.on('code-generator-ready', function () {
-        build();
       });
-
-      build();
     });
   }
 

lib/pipeline/pipeline.js

@@ -1,54 +1,25 @@
 const fs = require('../core/fs.js');
 const async = require('async');
+const child_process = require('child_process');
 const utils = require('../utils/utils.js');
-const webpack = require("webpack");
 const constants = require('../constants');
-const File = require('../core/file');
 
 require("babel-preset-react");
 require("babel-preset-es2015");
 require("babel-preset-es2016");
 require("babel-preset-es2017");
 
-let pipeline;
-
-// Override process.chdir so that we have a partial-implementation PWD for Windows
-const realChdir = process.chdir;
-process.chdir = (...args) => {
-  if (!process.env.PWD) {
-    process.env.PWD = process.cwd();
-  }
-  realChdir(...args);
-};
-
 class Pipeline {
 
   constructor(options) {
     this.buildDir = options.buildDir;
     this.contractsFiles = options.contractsFiles;
     this.assetFiles = options.assetFiles;
-    this.pipelinePlugins = options.pipelinePlugins;
+    this.events = options.events;
-    this.pluginImports = options.pluginImports;
+    this.logger = options.logger;
-    this.web3Location = options.web3Location;
+    this.normalizeInput = options.normalizeInput;
-    this.providerCode = options.providerCode;
+    this.plugins = options.plugins;
-
+    this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
-    this.interceptLogs();
-  }
-
-  interceptLogs() {
-    const context = {};
-    context.console = console;
-
-    context.console.log = this.log.bind(this, 'log');
-    context.console.warn = this.log.bind(this, 'warn');
-    context.console.info = this.log.bind(this, 'info');
-    context.console.debug = this.log.bind(this, 'debug');
-    context.console.trace = this.log.bind(this, 'trace');
-    context.console.dir = this.log.bind(this, 'dir');
-  }
-
-  log(type, ...messages) {
-    process.send({result: constants.pipeline.log, message: messages, type});
   }
 
   build(abi, contractsJSON, path, callback) {
@@ -66,7 +37,7 @@ class Pipeline {
         importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js');
         importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js');
 
-        self.pluginImports.forEach(function (importObject) {
+        self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) {
           let [importName, importLocation] = importObject;
           importsList[importName] = importLocation;
         });
@@ -83,14 +54,12 @@ class Pipeline {
         }, next);
       },
       function assetFileWrite(next) {
-        // limit:1 due to issues when downloading required files such as web3.js
+        async.eachOf(self.assetFiles, function (files, targetFile, cb) {
-        async.eachOfLimit(self.assetFiles, 1, function (files, targetFile, cb) {
+            async.map(files,
-            // limit:1 due to issues when downloading required files such as web3.js
-            async.mapLimit(files, 1,
               function (file, fileCb) {
-                file = new File(file); // Re-instantiate a File as through the process, we lose its prototype
+                self.logger.trace("reading " + file.filename);
-                console.trace("reading " + file.filename);
 
+                // Not a JS file
                 if (file.filename.indexOf('.js') < 0) {
                   return file.content(function (fileContent) {
                     self.runPlugins(file, fileContent, fileCb);
@@ -98,36 +67,24 @@ class Pipeline {
                 }
 
                 // JS files
-                let realCwd;
-
                 async.waterfall([
-
-                  function findImports(next) {
-                    self.webpackRun(file.filename, {}, false, importsList, false, next);
-                  },
-
-                  function changeCwd(next) {
-                    realCwd = utils.pwd();
-                    process.chdir(fs.embarkPath(''));
-                    next();
-                  },
-
                   function runWebpack(next) {
-                    self.webpackRun(file.filename, {}, true, importsList, true, next);
+                    const webpackProcess = child_process.fork(utils.joinPath(__dirname, 'webpackProcess.js'));
-                  },
+                    webpackProcess.send({action: constants.pipeline.init, options: {}});
+                    webpackProcess.send({action: constants.pipeline.build, file, importsList});
 
-                  function changeCwdBack(next) {
+                    webpackProcess.on('message', function (msg) {
-                    process.chdir(realCwd);
+                      if (msg.result === constants.pipeline.built) {
-                    next();
+                        webpackProcess.disconnect();
-                  },
+                        return next(msg.error);
-
+                      }
-                  function checkFile(next) {
+
-                    fs.access('./.embark/' + file.filename, (err) => {
+                      if (msg.result === constants.pipeline.log) {
-                      if (err) {
+                        if (self.logger[msg.type]) {
-                        console.error("couldn't find file: " + file.filename);
+                          return self.logger[msg.type](self.normalizeInput(msg.message));
-                        return next("couldn't find file: " + file.filename);
+                        }
+                        self.logger.debug(self.normalizeInput(msg.message));
                       }
-                      next();
                     });
                   },
 
@@ -146,8 +103,7 @@ class Pipeline {
 
                 ], function (err, contentFile) {
                   if (err) {
-                    process.chdir(realCwd);
+                    self.logger.error(err);
-                    console.error(err);
                     return fileCb(err);
                   }
 
@@ -156,10 +112,10 @@ class Pipeline {
               },
               function (err, contentFiles) {
                 if (err) {
-                  console.error('errors found while generating ' + targetFile);
+                  self.logger.error('errors found while generating ' + targetFile);
                 }
                 let dir = targetFile.split('/').slice(0, -1).join('/');
-                console.trace("creating dir " + self.buildDir + dir);
+                self.logger.trace("creating dir " + self.buildDir + dir);
                 fs.mkdirpSync(self.buildDir + dir);
 
                 // if it's a directory
@@ -172,7 +128,7 @@ class Pipeline {
 
                   async.each(contentFiles, function (file, mapCb) {
                     let filename = file.filename.replace(file.basedir + '/', '');
-                    console.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
+                    self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
 
                     fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb);
                   }, cb);
@@ -186,7 +142,7 @@ class Pipeline {
                   return file.content;
                 }).join("\n");
 
-                console.info("writing file " + (self.buildDir + targetFile).bold.dim);
+                self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
                 fs.writeFile(self.buildDir + targetFile, content, cb);
               }
             );
@@ -199,16 +155,10 @@ class Pipeline {
   runPlugins(file, fileContent, fileCb) {
     const self = this;
     if (self.pipelinePlugins.length <= 0) {
-      return fileCb(null, {
+      return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
-        content: fileContent,
-        filename: file.filename,
-        path: file.path,
-        basedir: file.basedir,
-        modified: true
-      });
     }
     async.eachSeries(self.pipelinePlugins,
-      function (plugin, pluginCB) {
+      function(plugin, pluginCB) {
         if (file.options && file.options.skipPipeline) {
           return pluginCB();
         }
@@ -219,85 +169,13 @@ class Pipeline {
       },
       function (err) {
         if (err) {
-          console.error(err.message);
+          self.logger.error(err.message);
         }
-        return fileCb(null, {
+        return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
-          content: fileContent,
-          filename: file.filename,
-          path: file.path,
-          basedir: file.basedir,
-          modified: true
-        });
       }
     );
   }
 
-  webpackRun(filename, options, includeModules, importsList, detectErrors, callback) {
-    let defaultOptions = {
-      entry: fs.dappPath(filename),
-      output: {
-        libraryTarget: 'umd',
-        path: fs.dappPath('.embark'),
-        filename: filename
-      },
-      resolve: {
-        alias: importsList,
-        modules: [
-          fs.embarkPath('node_modules'),
-          fs.dappPath('node_modules')
-        ]
-      },
-      externals: function (context, request, callback) {
-        callback();
-      }
-    };
-
-    let webpackOptions = utils.recursiveMerge(defaultOptions, options);
-
-    if (includeModules) {
-      webpackOptions.module = {
-        rules: [
-          {
-            test: /\.css$/,
-            use: [{loader: "style-loader"}, {loader: "css-loader"}]
-          },
-          {
-            test: /\.scss$/,
-            use: [{loader: "style-loader"}, {loader: "css-loader"}]
-          },
-          {
-            test: /\.(png|woff|woff2|eot|ttf|svg)$/,
-            loader: 'url-loader?limit=100000'
-          },
-          {
-            test: /\.js$/,
-            loader: "babel-loader",
-            exclude: /(node_modules|bower_components)/,
-            options: {
-              presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve),
-              plugins: ["babel-plugin-webpack-aliases"].map(require.resolve),
-              compact: false
-            }
-          }
-        ]
-      };
-    }
-
-    webpack(webpackOptions).run((err, stats) => {
-      if (err) {
-        console.error(err);
-      }
-      if (!detectErrors) {
-        return callback();
-      }
-
-      if (stats.hasErrors()) {
-        return callback(stats.toJson().errors.join("\n"));
-      }
-      callback();
-    });
-  }
-
   buildContracts(contractsJSON, callback) {
     fs.mkdirp(fs.dappPath(this.buildDir, 'contracts'), (err) => {
       if (err) {
@@ -337,21 +215,33 @@ class Pipeline {
     let code = "";
 
     async.waterfall([
-      function getImports(next) {
+      function getWeb3Location(next) {
-        self.web3Location = self.web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
+        self.events.request("version:get:web3", function(web3Version) {
-        code += "\nimport Web3 from '" + self.web3Location + "';\n";
+          if (web3Version === "1.0.0-beta") {
+            return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
+          } else {
+            self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
+              return next(null, fs.dappPath(location));
+            });
+          }
+        });
+      },
+      function getImports(web3Location, next) {
+        web3Location = web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
+        code += "\nimport Web3 from '" + web3Location + "';\n";
 
         code += "\n if (typeof web3 !== 'undefined') {";
         code += "\n } else {";
         code += "\n var web3 = new Web3();\n";
         code += "\n }";
 
-        code += self.providerCode;
+        self.events.request('provider-code', function(providerCode)  {
-        code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n";
+          code += providerCode;
-        code += "\nwindow.web3 = web3;\n";
+          code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n";
-        code += "\nexport default web3;\n";
+          code += "\nwindow.web3 = web3;\n";
-
+          code += "\nexport default web3;\n";
-        next();
+          next();
+        });
       },
       function makeDirectory(next) {
         fs.mkdirp(fs.dappPath(".embark"), (err, _result) => {
@@ -365,19 +255,4 @@ class Pipeline {
   }
 }
 
-process.on('message', (msg) => {
+module.exports = Pipeline;
-  if (msg.action === constants.pipeline.init) {
-    pipeline = new Pipeline(msg.options);
-    return process.send({result: constants.pipeline.initiated});
-  }
-
-  if (msg.action === constants.pipeline.build) {
-    return pipeline.build(msg.abi, msg.contractsJSON, msg.path, (err) => {
-      process.send({result: constants.pipeline.built, error: err});
-    });
-  }
-});
-
-process.on('exit', () => {
-  process.exit(0);
-});

lib/pipeline/watch.js

@@ -117,7 +117,7 @@ class Watch {
     this.logger.trace(files);
 
     let configWatcher = chokidar.watch(files, {
-      ignored: /[\/\\]\./, persistent: true, ignoreInitial: true, followSymlinks: true
+      ignored: /[\/\\]\.|tmp_/, persistent: true, ignoreInitial: true, followSymlinks: true
     });
     this.fileWatchers.push(configWatcher);
 

lib/pipeline/webpackProcess.js

@@ -0,0 +1,156 @@
+const async = require('async');
+const webpack = require('webpack');
+const utils = require('../utils/utils');
+const fs = require('../core/fs');
+const constants = require('../constants');
+const HardSourceWebpackPlugin = require('hard-source-webpack-plugin');
+
+// Override process.chdir so that we have a partial-implementation PWD for Windows
+const realChdir = process.chdir;
+process.chdir = (...args) => {
+  if (!process.env.PWD) {
+    process.env.PWD = process.cwd();
+  }
+  realChdir(...args);
+};
+
+let webpackProcess;
+
+class WebpackProcess {
+  constructor(_options) {
+    this.interceptLogs();
+  }
+
+  interceptLogs() {
+    const context = {};
+    context.console = console;
+
+    context.console.log = this.log.bind(this, 'log');
+    context.console.warn = this.log.bind(this, 'warn');
+    context.console.info = this.log.bind(this, 'info');
+    context.console.debug = this.log.bind(this, 'debug');
+    context.console.trace = this.log.bind(this, 'trace');
+    context.console.dir = this.log.bind(this, 'dir');
+  }
+
+  log(type, ...messages) {
+    if (messages[0].indexOf('HardSourceWebpackPlugin')) {
+      return;
+    }
+    process.send({result: constants.pipeline.log, message: messages, type});
+  }
+
+  build(file, importsList, callback) {
+    const self = this;
+    let realCwd;
+
+    async.waterfall([
+      function findImports(next) {
+        self.webpackRun(file.filename, {}, false, importsList, false, next);
+      },
+
+      function changeCwd(next) {
+        realCwd = utils.pwd();
+        process.chdir(fs.embarkPath(''));
+        next();
+      },
+
+      function runWebpack(next) {
+        self.webpackRun(file.filename, {}, true, importsList, true, next);
+      },
+
+      function changeCwdBack(next) {
+        process.chdir(realCwd);
+        next();
+      }
+    ], (err) => {
+      process.chdir(realCwd);
+      callback(err);
+    });
+  }
+
+
+  webpackRun(filename, options, includeModules, importsList, detectErrors, callback) {
+    let defaultOptions = {
+      entry: fs.dappPath(filename),
+      output: {
+        libraryTarget: 'umd',
+        path: fs.dappPath('.embark'),
+        filename: filename
+      },
+      resolve: {
+        alias: importsList,
+        modules: [
+          fs.embarkPath('node_modules'),
+          fs.dappPath('node_modules')
+        ]
+      },
+      externals: function (context, request, callback) {
+        callback();
+      },
+      plugins: [new HardSourceWebpackPlugin()]
+    };
+
+    let webpackOptions = utils.recursiveMerge(defaultOptions, options);
+
+    if (includeModules) {
+      webpackOptions.module = {
+        rules: [
+          {
+            test: /\.css$/,
+            use: [{loader: "style-loader"}, {loader: "css-loader"}]
+          },
+          {
+            test: /\.scss$/,
+            use: [{loader: "style-loader"}, {loader: "css-loader"}]
+          },
+          {
+            test: /\.(png|woff|woff2|eot|ttf|svg)$/,
+            loader: 'url-loader?limit=100000'
+          },
+          {
+            test: /\.js$/,
+            loader: "babel-loader",
+            exclude: /(node_modules|bower_components)/,
+            options: {
+              presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve),
+              plugins: ["babel-plugin-webpack-aliases"].map(require.resolve),
+              compact: false
+            }
+          }
+        ]
+      };
+    }
+
+    webpack(webpackOptions).run((err, stats) => {
+      if (err) {
+        console.error(err);
+      }
+      if (!detectErrors) {
+        return callback();
+      }
+
+      if (stats.hasErrors()) {
+        return callback(stats.toJson().errors.join("\n"));
+      }
+      callback();
+    });
+  }
+}
+
+process.on('message', (msg) => {
+  if (msg.action === constants.pipeline.init) {
+    webpackProcess = new WebpackProcess(msg.options);
+    return process.send({result: constants.pipeline.initiated});
+  }
+
+  if (msg.action === constants.pipeline.build) {
+    return webpackProcess.build(msg.file, msg.importsList, (err) => {
+      process.send({result: constants.pipeline.built, error: err});
+    });
+  }
+});
+
+process.on('exit', () => {
+  process.exit(0);
+});

package.json

@@ -44,11 +44,11 @@
     "follow-redirects": "^1.2.4",
     "fs-extra": "^2.0.0",
     "globule": "^1.1.0",
+    "hard-source-webpack-plugin": "^0.6.6",
     "http-shutdown": "^1.2.0",
     "ipfs-api": "17.2.4",
     "live-plugin-manager": "https://github.com/iurimatias/live-plugin-manager.git",
     "merge": "^1.2.0",
-    "mocha": "^2.2.5",
     "orbit-db": "^0.17.3",
     "parse-json": "^4.0.0",
     "promptly": "^2.1.0",