feat(modules/pipeline): move pipeline into its own module plugin

This is the first step of refactoring Embark's pipeline abstraction into
dedicated plugin modules that take advantage of Embark's event system.

With this commit we're moving `Pipeline` into `lib/modules/pipeline` and
introduce a new command handler `pipeline:build`. Embark's engine now
requests builds via this command handler.

Notice that `Watch` still lives in `lib/pipeline` as this is a step-by-step
refactoring to reduce chances of introducing regressions.

cmd/cmd_controller.js

@@ -419,12 +419,12 @@ class EmbarkController {
 
   ejectWebpack() {
     var fs = require('../lib/core/fs.js');
-    var embarkConfig = fs.embarkPath('lib/pipeline/webpack.config.js');
+    var embarkConfig = fs.embarkPath('lib/modules/pipeline/webpack.config.js');
     var dappConfig = fs.dappPath('webpack.config.js');
     fs.copyPreserve(embarkConfig, dappConfig);
     console.log(__('webpack config ejected to:').dim.yellow);
     console.log(`${dappConfig}`.green);
-    var embarkOverrides = fs.embarkPath('lib/pipeline/babel-loader-overrides.js');
+    var embarkOverrides = fs.embarkPath('lib/modules/pipeline/babel-loader-overrides.js');
     var dappOverrides = fs.dappPath('babel-loader-overrides.js');
     fs.copyPreserve(embarkOverrides, dappOverrides);
     console.log(__('webpack overrides ejected to:').dim.yellow);

lib/core/config.js

@@ -7,6 +7,8 @@ const deepEqual = require('deep-equal');
 const constants = require('../constants');
 const {canonicalHost, defaultHost} = require('../utils/host');
 
+const DEFAULT_CONFIG_PATH = 'config/';
+
 var Config = function(options) {
   const self = this;
   this.env = options.env;

lib/core/engine.js

@@ -103,21 +103,12 @@ class Engine {
 
   pipelineService(_options) {
     const self = this;
-    this.events.emit("status", "Building Assets");
+    this.registerModule('pipeline', {
-    const Pipeline = require('../pipeline/pipeline.js');
-    const pipeline = new Pipeline({
-      env: this.env,
-      buildDir: this.config.buildDir,
-      contractsFiles: this.config.contractsFiles,
-      assetFiles: this.config.assetFiles,
-      events: this.events,
-      logger: this.logger,
-      plugins: this.plugins,
       webpackConfigName: this.webpackConfigName
     });
     this.events.on('code-generator-ready', function () {
       self.events.request('code', function (abi, contractsJSON) {
-        pipeline.build(abi, contractsJSON, null, () => {
+        self.events.request('pipeline:build', {abi, contractsJSON}, () => {
           self.events.emit('outputDone');
         });
       });

lib/modules/pipeline/index.js

@@ -1,64 +1,60 @@
-const fs = require('../core/fs.js');
+const fs = require('../../core/fs.js');
 const async = require('async');
-const ProcessLauncher = require('../core/processes/processLauncher');
+const utils = require('../../utils/utils.js');
-const utils = require('../utils/utils.js');
+const ProcessLauncher = require('../../core/processes/processLauncher');
-const constants = require('../constants');
+const constants = require('../../constants');
 
 class Pipeline {
-  constructor(options) {
+  constructor(embark, options) {
-    this.env = options.env;
+    this.env = embark.config.env;
-    this.buildDir = options.buildDir;
+    this.buildDir  = embark.config.buildDir;
-    this.contractsFiles = options.contractsFiles;
+    this.contractsFiles = embark.config.contractsFiles;
-    this.assetFiles = options.assetFiles;
+    this.assetFiles = embark.config.assetFiles;
-    this.events = options.events;
+    this.events = embark.events;
-    this.logger = options.logger;
+    this.logger = embark.config.logger;
-    this.plugins = options.plugins;
+    this.plugins = embark.config.plugins;
     this.webpackConfigName = options.webpackConfigName;
     this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
     this.isFirstBuild = true;
 
+    this.events.setCommandHandler('pipeline:build', (options, callback) => this.build(callback));
     fs.removeSync(this.buildDir);
   }
 
-  build(abi, contractsJSON, path, callback) {
+  build(callback) {
     let self = this;
     const importsList = {};
     let placeholderPage;
 
-    if (!this.assetFiles || !Object.keys(this.assetFiles).length) {
+    if (!self.assetFiles || !Object.keys(self.assetFiles).length) {
       return self.buildContracts(callback);
     }
 
     async.waterfall([
-      function createPlaceholderPage(next){
+      function createPlaceholderPage(next) {
         if (self.isFirstBuild) {
           self.isFirstBuild = false;
           return next();
         }
         self.events.request('build-placeholder', next);
       },
-      function buildTheContracts(next) {
+      (next) => self.buildContracts(next),
-        self.buildContracts(next);
+      (next) => self.buildWeb3JS(next),
-      },
-      function buildWeb3(next) {
-        self.buildWeb3JS(next);
-      },
       function createImportList(next) {
         importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js');
         importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js');
         importsList["Embark/contracts"] = fs.dappPath(".embark/contracts", '');
 
-        self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) {
+        self.plugins.getPluginsProperty('imports', 'imports').forEach(importObject => {
           let [importName, importLocation] = importObject;
           importsList[importName] = importLocation;
         });
-
         next();
       },
       function writeContracts(next) {
         self.events.request('contracts:list', (_err, contracts) => {
           // ensure the .embark/contracts directory exists (create if not exists)
-          fs.mkdirp(fs.dappPath(".embark/contracts", ''), (err) => {
+          fs.mkdirp(fs.dappPath(".embark/contracts", ''), err => {
             if(err) return next(err);
 
             // Create a file .embark/contracts/index.js that requires all contract files
@@ -69,7 +65,7 @@ class Pipeline {
             importsHelperFile.write('module.exports = {\n');
 
             async.eachOf(contracts, (contract, idx, eachCb) => {
-              self.events.request('code-generator:contract', contract.className, (contractCode) => {
+              self.events.request('code-generator:contract', contract.className, contractCode => {
                 let filePath = fs.dappPath(".embark/contracts", contract.className + '.js');
                 importsList["Embark/contracts/" + contract.className] = filePath;
                 fs.writeFile(filePath, contractCode, eachCb);
@@ -78,7 +74,7 @@ class Pipeline {
                 importsHelperFile.write(`"${contract.className}": require('./${contract.className}').default`);
                 if(idx < contracts.length - 1) importsHelperFile.write(',\n'); // add a comma if we have more contracts to add
               });
-            }, function(){
+            }, () => {
               importsHelperFile.write('\n}'); // close the module.exports = {}
               importsHelperFile.close(next); // close the write stream
             });
@@ -87,8 +83,7 @@ class Pipeline {
       },
       function runWebpack(next) {
         self.logger.info(__(`running webpack with '${self.webpackConfigName}' config...`));
-        const assets = Object.keys(self.assetFiles)
+        const assets = Object.keys(self.assetFiles).filter(key => key.match(/\.js$/));
-          .filter(key => key.match(/\.js$/));
         if (!assets || !assets.length) {
           return next();
         }
@@ -100,7 +95,7 @@ class Pipeline {
           modulePath: utils.joinPath(__dirname, 'webpackProcess.js'),
           logger: self.logger,
           events: self.events,
-          exitCallback: function (code) {
+          exitCallback: code => {
             if (!built) {
               return next(`Webpack build exited with code ${code} before the process finished`);
             }
@@ -137,7 +132,7 @@ class Pipeline {
               files,
               function (file, fileCb) {
                 self.logger.trace("reading " + file.filename);
-                return file.content(function (fileContent) {
+                return file.content(fileContent => {
                   self.runPlugins(file, fileContent, fileCb);
                 });
               },
@@ -166,14 +161,14 @@ class Pipeline {
                   return;
                 }
 
-                let content = contentFiles.map(function (file) {
+                let content = contentFiles.map(file => {
                   if (file === undefined) {
                     return "";
                   }
                   return file.content;
                 }).join("\n");
 
-                if(new RegExp(/^index.html?/i).test(targetFile)){
+                if (new RegExp(/^index.html?/i).test(targetFile)) {
                   targetFile = targetFile.replace('index', 'index-temp');
                   placeholderPage = targetFile;
                 }
@@ -197,49 +192,22 @@ class Pipeline {
     ], callback);
   }
 
-  runPlugins(file, fileContent, fileCb) {
-    const self = this;
-    if (self.pipelinePlugins.length <= 0) {
-      return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
-    }
-    async.eachSeries(
-      self.pipelinePlugins,
-      function(plugin, pluginCB) {
-        if (file.options && file.options.skipPipeline) {
-          return pluginCB();
-        }
-
-        fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent});
-        file.modified = true;
-        pluginCB();
-      },
-      function (err) {
-        if (err) {
-          self.logger.error(err.message);
-        }
-        return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
-      }
-    );
-  }
-
   buildContracts(cb) {
     const self = this;
-
     async.waterfall([
       function makeDirectory(next) {
-        fs.mkdirp(fs.dappPath(self.buildDir, 'contracts'), (err, _result) => {
+        fs.mkdirp(fs.dappPath(self.buildDir, 'contracts'), err => next(err));
-          next(err);
-        });
       },
       function getContracts(next) {
-        self.events.request('contracts:list', (err, contracts) => {
+        self.events.request('contracts:list', next);
-          next(err, contracts);
-        });
       },
       function writeContractsJSON(contracts, next) {
-        async.each(contracts, (contract, eachCb) => {
+        async.each(contracts,(contract, eachCb) => {
-          fs.writeJson(fs.dappPath(self.buildDir, 'contracts', contract.className + ".json"), contract, {spaces: 2}, eachCb);
+          fs.writeJson(fs.dappPath(
-        }, () => { next(); });
+            self.buildDir,
+            'contracts', contract.className + '.json'
+          ), contract, {spaces: 2}, eachCb);
+        }, () => next());
       }
     ], cb);
   }
@@ -248,9 +216,7 @@ class Pipeline {
     const self = this;
     async.waterfall([
       function makeDirectory(next) {
-        fs.mkdirp(fs.dappPath(".embark"), (err, _result) => {
+        fs.mkdirp(fs.dappPath(".embark"), err => next(err));
-          next(err);
-        });
       },
       function getWeb3Code(next) {
         self.events.request('code-generator:web3js', next);
@@ -260,6 +226,28 @@ class Pipeline {
       }
     ], cb);
   }
+
+  runPlugins(file, fileContent, fileCb) {
+    const self = this;
+    if (self.pipelinePlugins.length <= 0) {
+      return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
+    }
+    async.eachSeries(self.pipelinePlugins, (plugin, pluginCB) => {
+      if (file.options && file.options.skipPipeline) {
+        return pluginCB();
+      }
+
+      fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent});
+      file.modified = true;
+      pluginCB();
+    }, err => {
+      if (err) {
+        self.logger.error(err.message);
+      }
+      return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
+    });
+  }
+
 }
 
 module.exports = Pipeline;

lib/modules/pipeline/webpackProcess.js

@@ -1,6 +1,6 @@
-const constants = require('../constants');
+const constants = require('../../constants');
-const fs = require('../core/fs');
+const fs = require('../../core/fs');
-const ProcessWrapper = require('../core/processes/processWrapper');
+const ProcessWrapper = require('../../core/processes/processWrapper');
 const webpack = require('webpack');
 const writeFile = require('util').promisify(require('fs').writeFile);
 const {errorMessage, recursiveMerge} = require('../utils/utils');
@@ -58,7 +58,7 @@ class WebpackProcess extends ProcessWrapper {
     }
 
     const dappConfigPath = fs.dappPath('webpack.config.js');
-    const defaultConfigPath = fs.embarkPath('lib/pipeline', 'webpack.config.js');
+    const defaultConfigPath = fs.embarkPath('lib/modules/pipeline', 'webpack.config.js');
 
     let config, configPath;
     try {

templates/demo/chains.json

@@ -1 +0,0 @@
-{}