Merge pull request #400 from embark-framework/features/write-files-process

Write files in a different process

lib/constants.json

@@ -15,5 +15,12 @@
   "events": {
     "contractFilesChanged": "contractFilesChanged",
     "contractConfigChanged": "contractConfigChanged"
+  },
+  "pipeline": {
+    "init": "init",
+    "build": "build",
+    "initiated": "initiated",
+    "built": "built",
+    "log": "log"
   }
 }

lib/core/engine.js

@@ -1,14 +1,17 @@
-let Web3 = require('web3');
+const Web3 = require('web3');
-let Events = require('./events.js');
+const Events = require('./events.js');
-let Logger = require('./logger.js');
+const Logger = require('./logger.js');
-let Config = require('./config.js');
+const Config = require('./config.js');
-let ContractsManager = require('../contracts/contracts.js');
+const ContractsManager = require('../contracts/contracts.js');
-let DeployManager = require('../contracts/deploy_manager.js');
+const DeployManager = require('../contracts/deploy_manager.js');
-let CodeGenerator = require('../contracts/code_generator.js');
+const CodeGenerator = require('../contracts/code_generator.js');
-let ServicesMonitor = require('./services_monitor.js');
+const ServicesMonitor = require('./services_monitor.js');
-let Pipeline = require('../pipeline/pipeline.js');
+const Watch = require('../pipeline/watch.js');
-let Watch = require('../pipeline/watch.js');
+const LibraryManager = require('../versions/library_manager.js');
-let LibraryManager = require('../versions/library_manager.js');
+const utils = require('../utils/utils');
+const constants = require('../constants');
+const async = require('async');
+const fs = require('../core/fs.js');
 
 class Engine {
   constructor(options) {
@@ -41,12 +44,7 @@ class Engine {
     }
   }
 
-  doInterceptLogs() {
+  normalizeInput(input) {
-    var self = this;
-    let context = {};
-    context.console = console;
-
-    let normalizeInput = function(input) {
     let args = Object.values(input);
     if (args.length === 0) {
       return "";
@@ -61,26 +59,31 @@ class Engine {
       if (Array.isArray(x)) { return x.join(','); }
       return x;
     }).toString() + ']');
-    };
+  }
+
+  doInterceptLogs() {
+    var self = this;
+    let context = {};
+    context.console = console;
 
     context.console.log  = function() {
-      self.logger.info(normalizeInput(arguments));
+      self.logger.info(self.normalizeInput(arguments));
     };
     context.console.warn  = function() {
-      self.logger.warn(normalizeInput(arguments));
+      self.logger.warn(self.normalizeInput(arguments));
     };
     context.console.info  = function() {
-      self.logger.info(normalizeInput(arguments));
+      self.logger.info(self.normalizeInput(arguments));
     };
     context.console.debug  = function() {
       // TODO: ue JSON.stringify
-      self.logger.debug(normalizeInput(arguments));
+      self.logger.debug(self.normalizeInput(arguments));
     };
     context.console.trace  = function() {
-      self.logger.trace(normalizeInput(arguments));
+      self.logger.trace(self.normalizeInput(arguments));
     };
     context.console.dir  = function() {
-      self.logger.dir(normalizeInput(arguments));
+      self.logger.dir(self.normalizeInput(arguments));
     };
   }
 
@@ -132,26 +135,80 @@ class Engine {
 
   pipelineService(_options) {
     let self = this;
-    this.events.emit("status", "Building Assets");
+    self.events.emit("status", "Building Assets");
-    let pipeline = new Pipeline({
+    const pipelineProcess = require('child_process').fork(utils.joinPath(__dirname, '../pipeline/pipeline.js'));
-      buildDir: this.config.buildDir,
+
-      contractsFiles: this.config.contractsFiles,
+    async.waterfall([
-      assetFiles: this.config.assetFiles,
+      function getWeb3Version(next) {
-      events: this.events,
+        self.events.request("version:get:web3", function(web3Version) {
-      logger: this.logger,
+          next(null, web3Version);
-      plugins: this.plugins
         });
-    this.events.on('code-generator-ready', function () {
+      },
-      self.events.request('code', function (abi, contractsJSON) {
+      function getWeb3Location(web3Version, next) {
-        self.currentAbi = abi;
+        if (web3Version === "1.0.0-beta") {
-        self.contractsJSON = contractsJSON;
+          return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
-        pipeline.build(abi, contractsJSON, null, function() {
+        } else {
+          self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
+            return next(err, fs.dappPath(location));
+          });
+        }
+      },
+      function getProviderCode(web3Location, next) {
+        self.events.once('code-generator-ready', function () {
+          self.events.request('provider-code', function(providerCode) {
+            next(null, web3Location, providerCode);
+          });
+        });
+      }
+    ], (err, web3Location, providerCode) => {
+      if (err) {
+        return self.logger.error(err);
+      }
+
+      // Setup Pipeline
+      pipelineProcess.send({action: constants.pipeline.init, options: {
+          buildDir: self.config.buildDir,
+          contractsFiles: self.config.contractsFiles,
+          assetFiles: self.config.assetFiles,
+          events: self.events,
+          pipelinePlugins: self.plugins.getPluginsFor('pipeline'),
+          pluginImports: self.plugins.getPluginsProperty('imports', 'imports'),
+          web3Location,
+          providerCode
+        }});
+
+      pipelineProcess.on('message', function (msg) {
+        if (msg.result === constants.pipeline.built) {
           if (self.watch) {
             self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched
           }
-          self.events.emit('outputDone');
+          if (msg.error) {
+            return self.logger.error(msg.error);
+          }
+          return self.events.emit('outputDone');
+        }
+
+        if (msg.result === constants.pipeline.log) {
+          if (self.logger[msg.type]) {
+            return self.logger[msg.type](self.normalizeInput(msg.message));
+          }
+          self.logger.debug(self.normalizeInput(msg.message));
+        }
       });
+
+      function build() {
+        self.events.request('code', function (abi, contractsJSON) {
+          self.currentAbi = abi;
+          self.contractsJSON = contractsJSON;
+          pipelineProcess.send({action: constants.pipeline.build, abi, contractsJSON, path: null});
         });
+      }
+
+      self.events.on('code-generator-ready', function () {
+        build();
+      });
+
+      build();
     });
   }
 

lib/core/fs.js

@@ -11,6 +11,10 @@ function mkdirp() {
   return fs.mkdirp.apply(fs.mkdirp, arguments);
 }
 
+function copy() {
+  return fs.copy.apply(fs.copy, arguments);
+}
+
 function copySync() {
   return fs.copySync.apply(fs.copySync, arguments);
 }
@@ -19,6 +23,10 @@ function appendFileSync() {
   return fs.appendFileSync.apply(fs.writeFileSync, arguments);
 }
 
+function writeFile() {
+  return fs.writeFile.apply(fs.writeFileSync, arguments);
+}
+
 function writeFileSync() {
   return fs.writeFileSync.apply(fs.writeFileSync, arguments);
 }
@@ -45,10 +53,18 @@ function writeJSONSync() {
   return fs.writeJSONSync.apply(fs.writeJSONSync, arguments);
 }
 
+function writeJson() {
+  return fs.writeJson.apply(fs.writeJson, arguments);
+}
+
 function existsSync() {
   return fs.existsSync.apply(fs.existsSync, arguments);
 }
 
+function access() {
+  return fs.access.apply(fs.access, arguments);
+}
+
 function removeSync() {
   return fs.removeSync.apply(fs.removeSync, arguments);
 }
@@ -67,18 +83,22 @@ function createWriteStream() {
 }
 
 module.exports = {
-  mkdirpSync: mkdirpSync,
+  mkdirpSync,
   mkdirp,
-  copySync: copySync,
+  copy,
+  copySync,
   readFile,
-  readFileSync: readFileSync,
+  readFileSync,
-  appendFileSync: appendFileSync,
+  appendFileSync,
-  writeFileSync: writeFileSync,
+  writeFile,
-  readJSONSync: readJSONSync,
+  writeFileSync,
-  writeJSONSync: writeJSONSync,
+  readJSONSync,
-  existsSync: existsSync,
+  writeJson,
-  removeSync: removeSync,
+  writeJSONSync,
-  embarkPath: embarkPath,
+  access,
-  dappPath: dappPath,
+  existsSync,
+  removeSync,
+  embarkPath,
+  dappPath,
   createWriteStream
 };

lib/pipeline/pipeline.js

@@ -1,57 +1,103 @@
-let fs = require('../core/fs.js');
+const fs = require('../core/fs.js');
-let async = require('async');
+const async = require('async');
-var utils = require('../utils/utils.js');
+const utils = require('../utils/utils.js');
 const webpack = require("webpack");
+const constants = require('../constants');
+const File = require('../core/file');
 
 require("babel-preset-react");
 require("babel-preset-es2015");
 require("babel-preset-es2016");
 require("babel-preset-es2017");
 
+let pipeline;
+
+// Override process.chdir so that we have a partial-implementation PWD for Windows
+const realChdir = process.chdir;
+process.chdir = (...args) => {
+  if (!process.env.PWD) {
+    process.env.PWD = process.cwd();
+  }
+  realChdir(...args);
+};
+
 class Pipeline {
 
   constructor(options) {
     this.buildDir = options.buildDir;
     this.contractsFiles = options.contractsFiles;
     this.assetFiles = options.assetFiles;
-    this.events = options.events;
+    this.pipelinePlugins = options.pipelinePlugins;
-    this.logger = options.logger;
+    this.pluginImports = options.pluginImports;
-    this.plugins = options.plugins;
+    this.web3Location = options.web3Location;
+    this.providerCode = options.providerCode;
+
+    this.interceptLogs();
+  }
+
+  interceptLogs() {
+    const context = {};
+    context.console = console;
+
+    context.console.log = this.log.bind(this, 'log');
+    context.console.warn = this.log.bind(this, 'warn');
+    context.console.info = this.log.bind(this, 'info');
+    context.console.debug = this.log.bind(this, 'debug');
+    context.console.trace = this.log.bind(this, 'trace');
+    context.console.dir = this.log.bind(this, 'dir');
+  }
+
+  log(type, ...messages) {
+    process.send({result: constants.pipeline.log, message: messages, type});
   }
 
   build(abi, contractsJSON, path, callback) {
     let self = this;
+    const importsList = {};
 
-    this.buildContracts(contractsJSON);
+    async.waterfall([
-
+      function buildTheContracts(next) {
-    self.buildWeb3JS(function() {
+        self.buildContracts(contractsJSON, next);
-
+      },
-      let importsList = {};
+      function buildWeb3(next) {
-
+        self.buildWeb3JS(next);
+      },
+      function createImportList(next) {
         importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js');
         importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js');
 
-      self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) {
+        self.pluginImports.forEach(function (importObject) {
           let [importName, importLocation] = importObject;
           importsList[importName] = importLocation;
         });
 
-      for (let contractName in contractsJSON) {
+        next();
-          let contractCode = self.buildContractJS(contractName);
+      },
+      function writeContracts(next) {
+        async.each(Object.keys(contractsJSON), (contractName, eachCb) => {
+          self.buildContractJS(contractName, (err, contractCode) => {
             let filePath = fs.dappPath(".embark", contractName + '.js');
-          fs.writeFileSync(filePath, contractCode);
             importsList["Embark/contracts/" + contractName] = filePath;
-      }
+            fs.writeFile(filePath, contractCode, eachCb);
-
+          });
+        }, next);
+      },
+      function assetFileWrite(next) {
         // limit:1 due to issues when downloading required files such as web3.js
         async.eachOfLimit(self.assetFiles, 1, function (files, targetFile, cb) {
             // limit:1 due to issues when downloading required files such as web3.js
             async.mapLimit(files, 1,
               function (file, fileCb) {
-            self.logger.trace("reading " + file.filename);
+                file = new File(file); // Re-instantiate a File as through the process, we lose its prototype
+                console.trace("reading " + file.filename);
 
-            if (file.filename.indexOf('.js') >= 0) {
+                if (file.filename.indexOf('.js') < 0) {
+                  return file.content(function (fileContent) {
+                    self.runPlugins(file, fileContent, fileCb);
+                  });
+                }
 
+                // JS files
                 let realCwd;
 
                 async.waterfall([
@@ -66,26 +112,6 @@ class Pipeline {
                     next();
                   },
 
-                //function findImportsPhase2(next) {
-                //  console.log("====> findImports_2");
-                //  self.webpackRun(file.filename, {
-                //    externals: function(context, request, callback) {
-                //      if (request === utils.joinPath(fs.dappPath(), file.filename)) {
-                //        callback();
-                //      } else {
-                //        //if (request.indexOf('Embark/contracts/') === 0) {
-                //        //  let contractName = request.split('/')[2];
-                //        //  let contractCode = self.buildContractJS(contractName);
-                //        //  let filePath = utils.joinPath(fs.dappPath(), ".embark", contractName + '.js');
-                //        //  fs.writeFileSync(filePath, contractCode);
-                //        //  importsList[request] = filePath;
-                //        //}
-                //        callback(null, "amd " + Math.random());
-                //      }
-                //    }
-                //  }, true, importsList, next);
-                //},
-
                   function runWebpack(next) {
                     self.webpackRun(file.filename, {}, true, importsList, true, next);
                   },
@@ -93,33 +119,47 @@ class Pipeline {
                   function changeCwdBack(next) {
                     process.chdir(realCwd);
                     next();
+                  },
+
+                  function checkFile(next) {
+                    fs.access('./.embark/' + file.filename, (err) => {
+                      if (err) {
+                        console.error("couldn't find file: " + file.filename);
+                        return next("couldn't find file: " + file.filename);
+                      }
+                      next();
+                    });
+                  },
+
+                  function readFile(next) {
+                    fs.readFile('./.embark/' + file.filename, (err, data) => {
+                      if (err) {
+                        return next(err);
+                      }
+                      next(null, data.toString());
+                    });
+                  },
+
+                  function runPluginsOnContent(fileContent, next) {
+                    self.runPlugins(file, fileContent, next);
                   }
 
-              ], function(err, _result) {
+                ], function (err, contentFile) {
                   if (err) {
                     process.chdir(realCwd);
-                  self.logger.error(err);
+                    console.error(err);
                     return fileCb(err);
                   }
-                if (!fs.existsSync('./.embark/' + file.filename)) {
+
-                  self.logger.error("couldn't find file: " + file.filename);
+                  fileCb(null, contentFile);
-                  return fileCb("couldn't find file: " + file.filename);
-                }
-                let fileContent = fs.readFileSync('./.embark/' + file.filename).toString();
-                self.runPlugins(file, fileContent, fileCb);
                 });
-            } else {
-              file.content(function(fileContent) {
-                self.runPlugins(file, fileContent, fileCb);
-              });
-            }
               },
               function (err, contentFiles) {
                 if (err) {
-              self.logger.warn('errors found while generating ' + targetFile);
+                  console.error('errors found while generating ' + targetFile);
                 }
                 let dir = targetFile.split('/').slice(0, -1).join('/');
-            self.logger.trace("creating dir " + self.buildDir + dir);
+                console.trace("creating dir " + self.buildDir + dir);
                 fs.mkdirpSync(self.buildDir + dir);
 
                 // if it's a directory
@@ -130,13 +170,15 @@ class Pipeline {
                     targetDir = targetDir + '/';
                   }
 
-              contentFiles.map(function (file) {
+                  async.each(contentFiles, function (file, mapCb) {
                     let filename = file.filename.replace(file.basedir + '/', '');
-                self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
+                    console.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
+
+                    fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb);
+                  }, cb);
+                  return;
+                }
 
-                fs.copySync(file.path, self.buildDir + targetDir + filename, {overwrite: true});
-              });
-            } else {
                 let content = contentFiles.map(function (file) {
                   if (file === undefined) {
                     return "";
@@ -144,26 +186,28 @@ class Pipeline {
                   return file.content;
                 }).join("\n");
 
-              self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
+                console.info("writing file " + (self.buildDir + targetFile).bold.dim);
-              fs.writeFileSync(self.buildDir + targetFile, content);
+                fs.writeFile(self.buildDir + targetFile, content, cb);
-            }
-            cb();
               }
             );
           },
-        function (_err, _results) {
+          next);
-          callback();
+      }
-        });
+    ], callback);
-    });
   }
 
   runPlugins(file, fileContent, fileCb) {
     const self = this;
-    let pipelinePlugins = self.plugins.getPluginsFor('pipeline');
+    if (self.pipelinePlugins.length <= 0) {
-    if (pipelinePlugins.length <= 0) {
+      return fileCb(null, {
-      return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
+        content: fileContent,
+        filename: file.filename,
+        path: file.path,
+        basedir: file.basedir,
+        modified: true
+      });
     }
-    async.eachSeries(pipelinePlugins,
+    async.eachSeries(self.pipelinePlugins,
       function (plugin, pluginCB) {
         if (file.options && file.options.skipPipeline) {
           return pluginCB();
@@ -175,9 +219,15 @@ class Pipeline {
       },
       function (err) {
         if (err) {
-          self.logger.error(err.message);
+          console.error(err.message);
         }
-        return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
+        return fileCb(null, {
+          content: fileContent,
+          filename: file.filename,
+          path: file.path,
+          basedir: file.basedir,
+          modified: true
+        });
       }
     );
   }
@@ -233,29 +283,39 @@ class Pipeline {
       };
     }
 
-    webpack(webpackOptions).run((_err, _stats) => {
+    webpack(webpackOptions).run((err, stats) => {
+      if (err) {
+        console.error(err);
+      }
       if (!detectErrors) {
         return callback();
       }
 
-      if (_stats.hasErrors()) {
+      if (stats.hasErrors()) {
-        return callback(_stats.toJson().errors.join("\n"));
+        return callback(stats.toJson().errors.join("\n"));
       }
       callback();
     });
   }
 
-  buildContracts(contractsJSON) {
+  buildContracts(contractsJSON, callback) {
-    fs.mkdirpSync(fs.dappPath(this.buildDir, 'contracts'));
+    fs.mkdirp(fs.dappPath(this.buildDir, 'contracts'), (err) => {
-
+      if (err) {
-    for (let className in contractsJSON) {
+        return callback(err);
+      }
+      async.each(Object.keys(contractsJSON), (className, eachCb) => {
         let contract = contractsJSON[className];
-      fs.writeJSONSync(fs.dappPath(this.buildDir, 'contracts', className + ".json"), contract, {spaces: 2});
+        fs.writeJson(fs.dappPath(this.buildDir, 'contracts', className + ".json"), contract, {spaces: 2}, eachCb);
-    }
+      }, callback);
+    });
   }
 
-  buildContractJS(contractName) {
+  buildContractJS(contractName, callback) {
-    let contractJSON = fs.readFileSync(fs.dappPath(this.buildDir, 'contracts', contractName + '.json')).toString();
+    fs.readFile(fs.dappPath(this.buildDir, 'contracts', contractName + '.json'), (err, contractJSON) => {
+      if (err) {
+        return callback(err);
+      }
+      contractJSON = contractJSON.toString();
 
       let contractCode = "";
       contractCode += "import web3 from 'Embark/web3';\n";
@@ -268,8 +328,8 @@ class Pipeline {
       contractCode += "\n});\n";
 
       contractCode += "export default " + contractName + ";\n";
-
+      callback(null, contractCode);
-    return contractCode;
+    });
   }
 
   buildWeb3JS(cb) {
@@ -277,44 +337,47 @@ class Pipeline {
     let code = "";
 
     async.waterfall([
-      function getWeb3Location(next) {
+      function getImports(next) {
-        self.events.request("version:get:web3", function(web3Version) {
+        self.web3Location = self.web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
-          if (web3Version === "1.0.0-beta") {
+        code += "\nimport Web3 from '" + self.web3Location + "';\n";
-            return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
-          } else {
-            self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
-              return next(null, fs.dappPath(location));
-            });
-          }
-        });
-      },
-      function getImports(web3Location, next) {
-        web3Location = web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
-        code += "\nimport Web3 from '" + web3Location + "';\n";
 
         code += "\n if (typeof web3 !== 'undefined') {";
         code += "\n } else {";
         code += "\n var web3 = new Web3();\n";
         code += "\n }";
 
-        self.events.request('provider-code', function(providerCode)  {
+        code += self.providerCode;
-          code += providerCode;
         code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n";
         code += "\nwindow.web3 = web3;\n";
         code += "\nexport default web3;\n";
+
         next();
+      },
+      function makeDirectory(next) {
+        fs.mkdirp(fs.dappPath(".embark"), (err, _result) => {
+          next(err);
         });
       },
       function writeFile(next) {
-        fs.mkdirpSync(fs.dappPath(".embark"));
+        fs.writeFile(fs.dappPath(".embark", 'web3_instance.js'), code, next);
-        fs.writeFileSync(fs.dappPath(".embark", 'web3_instance.js'), code);
-        next();
       }
-    ], function(_err, _result) {
+    ], cb);
-      cb();
+  }
+}
+
+process.on('message', (msg) => {
+  if (msg.action === constants.pipeline.init) {
+    pipeline = new Pipeline(msg.options);
+    return process.send({result: constants.pipeline.initiated});
+  }
+
+  if (msg.action === constants.pipeline.build) {
+    return pipeline.build(msg.abi, msg.contractsJSON, msg.path, (err) => {
+      process.send({result: constants.pipeline.built, error: err});
     });
   }
+});
 
-}
+process.on('exit', () => {
-
+  process.exit(0);
-module.exports = Pipeline;
+});