Merge pull request #720 from embark-framework/features/extensible-pipeline

extensible pipeline

cmd/cmd.js

@@ -1,8 +1,35 @@
 const program = require('commander');
 const EmbarkController = require('./cmd_controller.js');
 const i18n = require('../lib/core/i18n/i18n.js');
+const utils = require('../lib/utils/utils.js');
+
 let embark = new EmbarkController;
 
+// set PWD to process.cwd() since Windows doesn't have a value for PWD
+if (!process.env.PWD) {
+  process.env.PWD = process.cwd();
+}
+
+// set the anchor for embark's fs.dappPath()
+if (!process.env.DAPP_PATH) {
+  process.env.DAPP_PATH = process.env.PWD;
+}
+
+// set the anchor for embark's fs.embarkPath()
+if (!process.env.EMBARK_PATH) {
+  process.env.EMBARK_PATH = utils.joinPath(__dirname, '..');
+}
+
+// NOTE: setting NODE_PATH at runtime won't effect lookup behavior in the
+// current process, but will take effect in child processes; this enables
+// lookup of *global* embark's own node_modules from within dapp scripts (such
+// as an ejected webpack.config.js), making embark's dependencies trasitive
+// dependencies of a dapp without the dapp explicitly specifying embark as a
+// dependency in the dapp's package.json
+process.env.NODE_PATH = utils.joinPath(process.env.EMBARK_PATH, 'node_modules')
+  + (process.env.NODE_PATH ? require('path').delimiter : '')
+  + (process.env.NODE_PATH || '');
+
 class Cmd {
   constructor() {
     program.version(embark.version);
@@ -18,6 +45,7 @@ class Cmd {
     this.simulator();
     this.test();
     this.reset();
+    this.ejectWebpack();
     this.graph();
     this.upload();
     this.versionCmd();
@@ -99,6 +127,7 @@ class Cmd {
       .option('-c, --client [client]', __('Use a specific ethereum client or simulator (supported: %s)', 'geth, testrpc'))
       .option('--loglevel [loglevel]', __('level of logging to display') + ' ["error", "warn", "info", "debug", "trace"]', /^(error|warn|info|debug|trace)$/i, 'debug')
       .option('--locale [locale]', __('language to use (default: en)'))
+      .option('--pipeline [pipeline]', __('webpack config to use (default: production)'))
       .description(__('deploy and build dapp at ') + 'dist/ (default: development)')
       .action(function (env, _options) {
         i18n.setOrDetectLocale(_options.locale);
@@ -107,6 +136,7 @@ class Cmd {
         _options.logLevel = _options.loglevel; // fix casing
         _options.onlyCompile = _options.contracts;
         _options.client = _options.client || 'geth';
+        _options.webpackConfigName = _options.pipeline || 'production';
         embark.build(_options);
       });
   }
@@ -123,6 +153,7 @@ class Cmd {
       .option('--logfile [logfile]', __('filename to output logs (default: %s)', 'none'))
       .option('--loglevel [loglevel]', __('level of logging to display') + ' ["error", "warn", "info", "debug", "trace"]', /^(error|warn|info|debug|trace)$/i, 'debug')
       .option('--locale [locale]', __('language to use (default: en)'))
+      .option('--pipeline [pipeline]', __('webpack config to use (default: development)'))
       .description(__('run dapp (default: %s)', 'development'))
       .action(function (env, options) {
         i18n.setOrDetectLocale(options.locale);
@@ -135,7 +166,8 @@ class Cmd {
           runWebserver: !options.noserver,
           useDashboard: !options.nodashboard,
           logFile: options.logfile,
-          logLevel: options.loglevel
+          logLevel: options.loglevel,
+          webpackConfigName: options.pipeline || 'development'
         });
       });
   }
@@ -147,6 +179,7 @@ class Cmd {
       .option('--logfile [logfile]', __('filename to output logs (default: %s)', 'none'))
       .option('--loglevel [loglevel]', __('level of logging to display') + ' ["error", "warn", "info", "debug", "trace"]', /^(error|warn|info|debug|trace)$/i, 'debug')
       .option('--locale [locale]', __('language to use (default: en)'))
+      .option('--pipeline [pipeline]', __('webpack config to use (default: development)'))
       .description(__('Start the Embark console'))
       .action(function (env, options) {
         i18n.setOrDetectLocale(options.locale);
@@ -155,7 +188,8 @@ class Cmd {
           client: options.client || 'geth',
           locale: options.locale,
           logFile: options.logfile,
-          logLevel: options.loglevel
+          logLevel: options.loglevel,
+          webpackConfigName: options.pipeline || 'development'
         });
       });
   }
@@ -276,6 +310,15 @@ class Cmd {
       });
   }
 
+  ejectWebpack() {
+    program
+      .command('eject-webpack')
+      .description(__('copy the default webpack config into your dapp for customization'))
+      .action(function () {
+        embark.ejectWebpack();
+      });
+  }
+
   versionCmd() {
     program
     .command('version')

cmd/cmd_controller.js

@@ -5,11 +5,6 @@ require('colors');
 
 let version = require('../package.json').version;
 
-// Set PWD to CWD since Windows doesn't have a value for PWD
-if (!process.env.PWD) {
-  process.env.PWD = process.cwd();
-}
-
 class EmbarkController {
 
   constructor(options) {
@@ -84,7 +79,8 @@ class EmbarkController {
       logLevel: options.logLevel,
       context: self.context,
       useDashboard: options.useDashboard,
-      webServerConfig: webServerConfig
+      webServerConfig: webServerConfig,
+      webpackConfigName: options.webpackConfigName
     });
     engine.init();
 
@@ -191,7 +187,8 @@ class EmbarkController {
       logger: options.logger,
       config: options.config,
       plugins: options.plugins,
-      context: this.context
+      context: this.context,
+      webpackConfigName: options.webpackConfigName
     });
     engine.init();
 
@@ -259,7 +256,8 @@ class EmbarkController {
       logFile: options.logFile,
       logLevel: options.logLevel,
       context: this.context,
-      ipcRole: 'client'
+      ipcRole: 'client',
+      webpackConfigName: options.webpackConfigName
     });
     engine.init();
     async.waterfall([
@@ -404,10 +402,33 @@ class EmbarkController {
     var fs = require('../lib/core/fs.js');
     fs.removeSync('./chains.json');
     fs.removeSync('.embark/');
+    fs.removeSync('node_modules/.cache');
     fs.removeSync('dist/');
     console.log(__("reset done!").green);
   }
 
+  ejectWebpack() {
+    var fs = require('../lib/core/fs.js');
+    var dappConfig = fs.dappPath('webpack.config.js');
+    var embarkConfig = fs.embarkPath('lib/pipeline', 'webpack.config.js');
+    let ext = 1;
+    let dappConfigOld = dappConfig;
+    while (fs.existsSync(dappConfigOld)) {
+      dappConfigOld = dappConfig + `.${ext}`;
+      ext++;
+    }
+    if (dappConfigOld !== dappConfig) {
+      fs.copySync(dappConfig, dappConfigOld);
+      console.warn(`${dappConfig}`.yellow);
+      console.warn(__('copied to').dim.yellow);
+      console.warn(`${dappConfigOld}\n`.yellow);
+    }
+    fs.copySync(embarkConfig, dappConfig);
+    console.log(`${embarkConfig}`.green);
+    console.log(__('copied to').dim.green);
+    console.log(`${dappConfig}`.green);
+  }
+
   upload(options) {
     this.context = options.context || [constants.contexts.upload, constants.contexts.build];
 

lib/core/config.js

@@ -169,7 +169,7 @@ Config.prototype._getFileOrOject = function(object, filePath, property) {
   if (typeof (this.configDir) === 'object') {
     return this.configDir[property];
   }
-  return this.configDir + filePath;
+  return utils.joinPath(this.configDir, filePath);
 };
 
 Config.prototype.loadBlockchainConfigFile = function() {

lib/core/engine.js

@@ -18,6 +18,7 @@ class Engine {
     this.useDashboard = options.useDashboard;
     this.webServerConfig = options.webServerConfig;
     this.ipcRole = options.ipcRole;
+    this.webpackConfigName = options.webpackConfigName;
   }
 
   init(_options) {
@@ -102,7 +103,8 @@ class Engine {
       assetFiles: this.config.assetFiles,
       events: this.events,
       logger: this.logger,
-      plugins: this.plugins
+      plugins: this.plugins,
+      webpackConfigName: this.webpackConfigName
     });
     this.events.on('code-generator-ready', function () {
       self.events.request('code', function (abi, contractsJSON) {

lib/core/fs.js

@@ -77,13 +77,21 @@ function removeSync() {
   return fs.removeSync.apply(fs.removeSync, arguments);
 }
 
-// returns embarks root directory
+function anchoredPath(envAnchor, ...args) {
-function embarkPath(fileOrDir) {
+  const anchor = process.env[envAnchor];
-  return utils.joinPath(__dirname, '/../../', fileOrDir);
+  if (!anchor) {
+    console.error(`process.env.${envAnchor} was not set`.bold.red);
+    process.exit(1);
+  }
+  return utils.joinPath(anchor, ...args);
+}
+
+function embarkPath() {
+  return anchoredPath('EMBARK_PATH', ...arguments);
 }
 
 function dappPath() {
-  return utils.joinPath(utils.pwd(), ...arguments);
+  return anchoredPath('DAPP_PATH', ...arguments);
 }
 
 function createWriteStream() {

lib/core/plugins.js

@@ -1,6 +1,5 @@
 const async = require('async');
 var Plugin = require('./plugin.js');
-var utils = require('../utils/utils.js');
 var fs = require('../core/fs.js');
 
 var Plugins = function(options) {
@@ -75,7 +74,7 @@ Plugins.prototype.loadInternalPlugin = function(pluginName, pluginConfig) {
 };
 
 Plugins.prototype.loadPlugin = function(pluginName, pluginConfig) {
-  var pluginPath = utils.joinPath(utils.pwd(), 'node_modules', pluginName);
+  var pluginPath = fs.dappPath('node_modules', pluginName);
   var plugin = require(pluginPath);
 
   var pluginWrapper = new Plugin({

lib/core/processes/processWrapper.js

@@ -5,11 +5,6 @@ process.on('uncaughtException', function(e){
 const constants = require('../../constants');
 const Events = require('./eventsWrapper');
 
-// Set PWD to CWD since Windows doesn't have a value for PWD
-if (!process.env.PWD) {
-  process.env.PWD = process.cwd();
-}
-
 class ProcessWrapper {
 
   /**

lib/pipeline/pipeline.js

@@ -14,6 +14,7 @@ class Pipeline {
     this.events = options.events;
     this.logger = options.logger;
     this.plugins = options.plugins;
+    this.webpackConfigName = options.webpackConfigName;
     this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
   }
 
@@ -30,7 +31,7 @@ class Pipeline {
       function createPlaceholderPage(next){
         self.events.request('embark-building-placeholder', (html) => {
           fs.mkdirpSync(self.buildDir); // create dist/ folder if not already exists
-          fs.writeFile(self.buildDir + 'index.html', html, next);
+          fs.writeFile(utils.joinPath(self.buildDir, 'index.html'), html, next);
         });
       },
       function buildTheContracts(next) {
@@ -81,66 +82,57 @@ class Pipeline {
           });
         });
       },
+      function runWebpack(next) {
+        self.logger.info(__(`running webpack with '${self.webpackConfigName}' config...`));
+        Object.keys(self.assetFiles)
+          .filter(key => key.match(/\.js?$/))
+          .forEach(key => {
+            self.logger.info(__("writing file") + " " + (utils.joinPath(self.buildDir, key)).bold.dim);
+          });
+        let built = false;
+        const webpackProcess = new ProcessLauncher({
+          modulePath: utils.joinPath(__dirname, 'webpackProcess.js'),
+          logger: self.logger,
+          events: self.events,
+          exitCallback: function (code) {
+            if (!built) {
+              return next(`Webpack build exited with code ${code} before the process finished`);
+            }
+            if (code) {
+              self.logger.error(__('Webpack build process exited with code ', code));
+            }
+          }
+        });
+        webpackProcess.send({action: constants.pipeline.init, options: {webpackConfigName: self.webpackConfigName}});
+        webpackProcess.send({action: constants.pipeline.build, assets: self.assetFiles, importsList});
+
+        webpackProcess.once('result', constants.pipeline.built, (msg) => {
+          built = true;
+          webpackProcess.kill();
+          return next(msg.error);
+        });
+      },
       function assetFileWrite(next) {
-        async.eachOf(self.assetFiles, function (files, targetFile, cb) {
+        async.eachOf(
-            async.map(files,
+          // assetFileWrite should not process .js files
+          Object.keys(self.assetFiles)
+            .filter(key => !key.match(/\.js?$/))
+            .reduce((obj, key) => {
+              obj[key] = self.assetFiles[key];
+              return obj;
+            }, {}),
+          function (files, targetFile, cb) {
+            const isDir = targetFile.slice(-1) === '/' || targetFile.slice(-1) === '\\' || targetFile.indexOf('.') === -1;
+            // if it's not a directory
+            if (!isDir) {
+              self.logger.info(__("writing file") + " " + (utils.joinPath(self.buildDir, targetFile)).bold.dim);
+            }
+            async.map(
+              files,
               function (file, fileCb) {
                 self.logger.trace("reading " + file.filename);
-
+                return file.content(function (fileContent) {
-                // Not a JS file
+                  self.runPlugins(file, fileContent, fileCb);
-                if (file.filename.indexOf('.js') < 0) {
-                  return file.content(function (fileContent) {
-                    self.runPlugins(file, fileContent, fileCb);
-                  });
-                }
-
-                // JS files
-                async.waterfall([
-                  function runWebpack(next) {
-                    let built = false;
-                    const webpackProcess = new ProcessLauncher({
-                      modulePath: utils.joinPath(__dirname, 'webpackProcess.js'),
-                      logger: self.logger,
-                      events: self.events,
-                      exitCallback: function (code) {
-                        if (!built) {
-                          return next(`File building of ${file.filename} exited with code ${code} before the process finished`);
-                        }
-                        if (code) {
-                          self.logger(__('File building process exited with code ', code));
-                        }
-                      }
-                    });
-                    webpackProcess.send({action: constants.pipeline.init, options: {env: self.env}});
-                    webpackProcess.send({action: constants.pipeline.build, file, importsList});
-
-                    webpackProcess.once('result', constants.pipeline.built, (msg) => {
-                      built = true;
-                      webpackProcess.kill();
-                      return next(msg.error);
-                    });
-                  },
-
-                  function readFile(next) {
-                    fs.readFile('./.embark/' + file.filename, (err, data) => {
-                      if (err) {
-                        return next(err);
-                      }
-                      next(null, data.toString());
-                    });
-                  },
-
-                  function runPluginsOnContent(fileContent, next) {
-                    self.runPlugins(file, fileContent, next);
-                  }
-
-                ], function (err, contentFile) {
-                  if (err) {
-                    self.logger.error(err.message || err);
-                    return fileCb(err);
-                  }
-
-                  fileCb(null, contentFile);
                 });
               },
               function (err, contentFiles) {
@@ -148,22 +140,22 @@ class Pipeline {
                   self.logger.error(__('errors found while generating') + ' ' + targetFile);
                 }
                 let dir = targetFile.split('/').slice(0, -1).join('/');
-                self.logger.trace("creating dir " + self.buildDir + dir);
+                self.logger.trace("creating dir " + utils.joinPath(self.buildDir, dir));
-                fs.mkdirpSync(self.buildDir + dir);
+                fs.mkdirpSync(utils.joinPath(self.buildDir, dir));
 
                 // if it's a directory
-                if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
+                if (isDir) {
                   let targetDir = targetFile;
 
                   if (targetDir.slice(-1) !== '/') {
                     targetDir = targetDir + '/';
                   }
 
-                  async.each(contentFiles, function (file, mapCb) {
+                  async.each(contentFiles, function (file, eachCb) {
                     let filename = file.filename.replace(file.basedir + '/', '');
-                    self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
+                    self.logger.info("writing file " + (utils.joinPath(self.buildDir, targetDir, filename)).bold.dim);
 
-                    fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb);
+                    fs.copy(file.path, utils.joinPath(self.buildDir, targetDir, filename), {overwrite: true}, eachCb);
                   }, cb);
                   return;
                 }
@@ -175,20 +167,20 @@ class Pipeline {
                   return file.content;
                 }).join("\n");
 
-                self.logger.info(__("writing file") + " " + (self.buildDir + targetFile).bold.dim);
                 if(new RegExp(/^index.html?/i).test(targetFile)){
                   targetFile = targetFile.replace('index', 'index-temp');
                   placeholderPage = targetFile;
                 }
-                fs.writeFile(self.buildDir + targetFile, content, cb);
+                fs.writeFile(utils.joinPath(self.buildDir, targetFile), content, cb);
               }
             );
           },
-          next);
+          next
+        );
       },
       function removePlaceholderPage(next){
-        let placeholderFile = self.buildDir + placeholderPage;
+        let placeholderFile = utils.joinPath(self.buildDir, placeholderPage);
-        fs.access(self.buildDir + placeholderPage, (err) => {
+        fs.access(utils.joinPath(self.buildDir, placeholderPage), (err) => {
           if (err) return next(); // index-temp doesn't exist, do nothing
 
           // rename index-temp.htm/l to index.htm/l, effectively replacing our placeholder page
@@ -204,7 +196,8 @@ class Pipeline {
     if (self.pipelinePlugins.length <= 0) {
       return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
     }
-    async.eachSeries(self.pipelinePlugins,
+    async.eachSeries(
+      self.pipelinePlugins,
       function(plugin, pluginCB) {
         if (file.options && file.options.skipPipeline) {
           return pluginCB();

lib/pipeline/webpack.config.js

@@ -0,0 +1,176 @@
+// some packages, plugins, and presets referenced/required in this webpack
+// config are deps of embark and will be transitive dapp deps unless specified
+// in the dapp's own package.json
+
+// embark modifies process.env.NODE_PATH so that when running dapp scripts in
+// embark's child processes, embark's own node_modules directory will be
+// searched by node's require(); however, webpack and babel do not directly
+// support NODE_PATH, so modules such as babel plugins and presets must be
+// resolved with require.resolve(); that is only necessary if a plugin/preset
+// is in embark's node_modules vs. the dapp's node_modules
+
+const cloneDeep = require('lodash.clonedeep');
+const CompressionPlugin = require('compression-webpack-plugin');
+const glob = require('glob');
+const HardSourceWebpackPlugin = require('hard-source-webpack-plugin');
+const path = require('path');
+
+const dappPath = process.env.DAPP_PATH;
+const embarkPath = process.env.EMBARK_PATH;
+
+const embarkAliases = require(path.join(dappPath, '.embark/embark-aliases.json'));
+const embarkAssets = require(path.join(dappPath, '.embark/embark-assets.json'));
+const embarkNodeModules = path.join(embarkPath, 'node_modules');
+const embarkJson = require(path.join(dappPath, 'embark.json'));
+
+const buildDir = path.join(dappPath, embarkJson.buildDir);
+
+// it's important to `embark reset` if a pkg version is specified in
+// embark.json and changed/removed later, otherwise pkg resolution may behave
+// unexpectedly
+let versions;
+try {
+  versions = glob.sync(path.join(dappPath, '.embark/versions/*/*'));
+} catch (e) {
+  versions = [];
+}
+
+const entry = Object.keys(embarkAssets)
+  .filter(key => key.match(/\.js?$/))
+  .reduce((obj, key) => {
+    // webpack entry paths should start with './' if they're relative to the
+    // webpack context; embark.json "app" keys correspond to lists of .js
+    // source paths relative to the top-level dapp dir and may be missing the
+    // leading './'
+    obj[key] = embarkAssets[key]
+      .map(file => {
+        let file_path = file.path;
+        if (!file.path.match(/^\.\//)) {
+          file_path = './' + file_path;
+        }
+        return file_path;
+      });
+    return obj;
+  }, {});
+
+function resolve(pkgName) {
+  if (Array.isArray(pkgName)) {
+    const _pkgName = pkgName[0];
+    pkgName[0] = require.resolve(_pkgName);
+    return pkgName;
+  }
+  return require.resolve(pkgName);
+}
+
+// base config
+// -----------------------------------------------------------------------------
+
+const base = {
+  context: dappPath,
+  entry: entry,
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        use: [{loader: 'style-loader'}, {loader: 'css-loader'}]
+      },
+      {
+        test: /\.scss$/,
+        use: [{loader: 'style-loader'}, {loader: 'css-loader'}]
+      },
+      {
+        test: /\.(png|woff|woff2|eot|ttf|svg)$/,
+        loader: 'url-loader?limit=100000'
+      },
+      {
+        test: /\.jsx?$/,
+        loader: 'babel-loader',
+        exclude: /(node_modules|bower_components|\.embark[\\/]versions)/,
+        options: {
+          plugins: [
+            [
+              'babel-plugin-module-resolver', {
+                'alias': embarkAliases
+              }
+            ],
+            [
+              '@babel/plugin-transform-runtime', {
+                corejs: 2,
+                useESModules: true
+              }
+            ]
+          ].map(resolve),
+          presets: [
+            [
+              '@babel/preset-env', {
+                modules: false,
+                targets: {
+                  browsers: ['last 1 version', 'not dead', '> 0.2%']
+                }
+              }
+            ],
+            '@babel/preset-react'
+          ].map(resolve)
+        }
+      }
+    ]
+  },
+  output: {
+    filename: (chunkData) => chunkData.chunk.name,
+    // globalObject workaround for node-compatible UMD builds with webpack 4
+    // see: https://github.com/webpack/webpack/issues/6522#issuecomment-371120689
+    globalObject: 'typeof self !== \'undefined\' ? self : this',
+    libraryTarget: 'umd',
+    path: buildDir
+  },
+  plugins: [new HardSourceWebpackPlugin()],
+  // profiling and generating verbose stats increases build time; if stats
+  // are generated embark will write the output to:
+  //   path.join(dappPath, '.embark/stats.[json,report]')
+  // to visualize the stats info in a browser run:
+  //   npx webpack-bundle-analyzer .embark/stats.json <buildDir>
+  profile: true, stats: 'verbose',
+  resolve: {
+    alias: embarkAliases,
+    modules: [
+      ...versions,
+      'node_modules',
+      embarkNodeModules
+    ]
+  },
+  resolveLoader: {
+    modules: [
+      'node_modules',
+      embarkNodeModules
+    ]
+  }
+};
+
+// development config
+// -----------------------------------------------------------------------------
+
+const development = cloneDeep(base);
+// full source maps increase build time but are useful during dapp development
+development.devtool = 'source-map';
+development.mode = 'development';
+// alternatively:
+// development.mode = 'none';
+development.name = 'development';
+const devBabelLoader = development.module.rules[3];
+devBabelLoader.options.compact = false;
+
+// production config
+// -----------------------------------------------------------------------------
+
+const production = cloneDeep(base);
+production.mode = 'production';
+production.name = 'production';
+production.plugins.push(new CompressionPlugin());
+
+// export a list of named configs
+// -----------------------------------------------------------------------------
+
+module.exports = [
+  development,
+  production
+];

lib/pipeline/webpackProcess.js

@@ -1,208 +1,104 @@
-const async = require('async');
-const webpack = require('webpack');
-const utils = require('../utils/utils');
-const fs = require('../core/fs');
 const constants = require('../constants');
-const HardSourceWebpackPlugin = require('hard-source-webpack-plugin');
+const fs = require('../core/fs');
 const ProcessWrapper = require('../core/processes/processWrapper');
-const path = require('path');
+const webpack = require('webpack');
-const glob = require('glob');
+const writeFile = require('util').promisify(require('fs').writeFile);
+const {errorMessage} = require('../utils/utils');
 
 let webpackProcess;
 
 class WebpackProcess extends ProcessWrapper {
   constructor(options) {
     super(options);
-    this.env = options.env;
+    this.webpackConfigName = options.webpackConfigName;
   }
 
-  build(file, importsList, callback) {
+  async build(assets, importsList, callback) {
-    const self = this;
+    try {
-    let realCwd;
+      await this.webpackRun(assets, importsList, callback);
-
+    } catch (e) {
-    async.waterfall([
+      callback(errorMessage(e));
-      function changeCwd(next) {
+    }
-        realCwd = utils.pwd();
-        process.chdir(fs.embarkPath(''));
-        next();
-      },
-
-      function runWebpack(next) {
-        self.webpackRun(file.filename, {}, true, importsList, true, realCwd, next);
-      },
-
-      function changeCwdBack(next) {
-        process.chdir(realCwd);
-        next();
-      }
-    ], (err) => {
-      process.chdir(realCwd);
-      callback(err);
-    });
   }
 
-  webpackRun(filename, options, includeModules, importsList, detectErrors, realCwd, callback) {
+  async webpackRun(assets, importsList, callback) {
-    const self = this;
+    try {
-    glob(fs.dappPath('.embark/versions/*/*'), (err, files) => {
+      await writeFile(
-      let versions;
+        fs.dappPath('.embark/embark-aliases.json'),
-      if (err) {
+        JSON.stringify(importsList)
-        console.error(err);
+      );
-        versions = [];
+      await writeFile(
+        fs.dappPath('.embark/embark-assets.json'),
+        JSON.stringify(assets)
+      );
+    } catch (e) {
+      return callback(errorMessage(e));
+    }
+
+    const dappConfigPath = fs.dappPath('webpack.config.js');
+    const defaultConfigPath = fs.embarkPath('lib/pipeline', 'webpack.config.js');
+
+    let config, configPath;
+    try {
+      if (fs.existsSync(dappConfigPath)) {
+        configPath = dappConfigPath;
+        delete require.cache[configPath];
       } else {
-        versions = files;
+        configPath = defaultConfigPath;
       }
-      let defaultOptions = {
+      config = require(configPath);
-        mode: self.env === 'production' ? 'production' : 'none',
+      // valid config types: https://webpack.js.org/configuration/configuration-types/
-        // devtool: self.env === 'development' ? 'source-map' : false,
+      // + function that returns a config object
-        // pipeline would need to copy .map files to dist/ target dir
+      // + function that returns a promise for a config object
-        // note: generating full source maps ('source-map') roughly doubles build time
+      // + array of named config objects
-        entry: fs.dappPath(filename),
+      // + config object
-        output: {
+      if (typeof config === 'function') {
-          globalObject: 'typeof self !== \'undefined\' ? self : this',
+        config = await config(this.webpackConfigName);
-          libraryExport: 'default',
+      } else if (Array.isArray(config)) {
-          libraryTarget: 'umd',
+        config = config.filter(cfg => cfg.name === this.webpackConfigName);
-          path: fs.dappPath('.embark'),
+        if (!config.length) {
-          filename: filename,
+          return callback(`no webpack config has the name '${this.webpackConfigName}'`);
-          umdNamedDefine: true
-        },
-        // profile: true,
-        // stats: 'verbose',
-        // note: generating and writing to disk verbose stats increases build time
-        resolve: {
-          alias: importsList,
-          modules: [
-            fs.dappPath('node_modules'),
-            ...versions,
-            fs.embarkPath('node_modules')
-          ]
-        },
-        plugins: [
-          new HardSourceWebpackPlugin({
-            cacheDirectory: fs.dappPath('node_modules/.cache/hard-source'),
-            // ufglify (wp mode: production) will still save its cache in embark's node_modules/.cache/
-            environmentHash: {
-              root: fs.dappPath()
-            }
-          }),
-          new HardSourceWebpackPlugin.ExcludeModulePlugin(
-            [{test: /app[\\/]|contracts[\\/]/}]
-          )
-        ]
-      };
-
-      let webpackOptions = utils.recursiveMerge(defaultOptions, options);
-
-      if (includeModules) {
-        webpackOptions.module = {
-          rules: [
-            {
-              test: /\.css$/,
-              use: [{loader: "style-loader"}, {loader: "css-loader"}]
-            },
-            {
-              test: /\.scss$/,
-              use: [{loader: "style-loader"}, {loader: "css-loader"}]
-            },
-            {
-              test: /\.(png|woff|woff2|eot|ttf|svg)$/,
-              loader: 'url-loader?limit=100000'
-            },
-            {
-              test: /\.js$/,
-              loader: "babel-loader",
-              exclude: /(node_modules|bower_components|\.embark[\\/]versions)/,
-              options: {
-                presets: [
-                  [
-                    "@babel/preset-env", {
-                      modules: false,
-                      targets: {
-                        browsers: ["last 1 version", "not dead", "> 0.2%"]
-                      }
-                    }
-                  ],
-                  "@babel/preset-react"
-                ].map(pkg => {
-                  if (Array.isArray(pkg)) {
-                    let _pkg = pkg[0];
-                    pkg[0] = require.resolve(_pkg);
-                    return pkg;
-                  } else {
-                    return require.resolve(pkg);
-                  }
-                }),
-                plugins: [
-                  "babel-plugin-webpack-aliases",
-                  [
-                    "@babel/plugin-transform-runtime", {
-                      corejs: 2,
-                      useESModules: true
-                    }
-                  ]
-                ].map(pkg => {
-                  if (Array.isArray(pkg)) {
-                    let _pkg = pkg[0];
-                    pkg[0] = require.resolve(_pkg);
-                    return pkg;
-                  } else {
-                    return require.resolve(pkg);
-                  }
-                }),
-                compact: false
-              }
-            }
-          ]
-        };
-
-        let dappBabelrc = path.join(realCwd, '.babelrc');
-        if (fs.existsSync(dappBabelrc)) {
-          webpackOptions.module.rules[3].options.extends = dappBabelrc;
         }
+        if (config.length > 1) {
+          console.warn(`detected ${config.length} webpack configs having the name '${this.webpackConfigName}', using the first one`);
+        }
+        config = config[0];
       }
+    } catch (e) {
+      console.error(`error while loading webpack config ${configPath}`);
+      callback(errorMessage(e));
+    }
 
-      webpack(webpackOptions).run((err, stats) => {
+    if (typeof config !== 'object' || config === null) {
-        async.waterfall([
+      return callback('bad webpack config, the resolved config was null or not an object');
-          function checkStatsError(next) {
+    }
-            if (err) {
+
-              console.error(err);
+    webpack(config).run(async (err, stats) => {
-              return next(err);
+      if (err) {
-            }
+        return callback(errorMessage(err));
-            if (!detectErrors) {
+      }
-              return next();
+      if (!config.stats || config.stats === 'none') {
-            }
+        return callback();
-            if (stats.hasErrors()) {
+      }
-              return next(
+      try {
-                stats.toJson(webpackOptions.stats).errors.join("\n")
+        this._log('info', 'writing file '+ ('.embark/stats.report').bold.dim);
-              );
+        await writeFile(
-            }
+          fs.dappPath('.embark/stats.report'),
-            next();
+          stats.toString(config.stats)
-          }//,
+        );
-          // function writeStatsReport(next) {
+        this._log('info', 'writing file ' + ('.embark/stats.json').bold.dim);
-          //   if (detectErrors) {
+        await writeFile(
-          //     self._log('info', 'writing file '+ ('.embark/stats.report').bold.dim);
+          fs.dappPath('.embark/stats.json'),
-          //   }
+          JSON.stringify(stats.toJson(config.stats))
-          //   fs.writeFile(
+        );
-          //     path.join(fs.dappPath('.embark'), 'stats.report'),
+        if (stats.hasErrors()) {
-          //     stats.toString(webpackOptions.stats),
+          const errors = stats.toJson(config.stats).errors.join('\n');
-          //     next
+          return callback(errors);
-          //   );
+        }
-          // },
+        callback();
-          // function writeStatsJSON(next) {
+      } catch (e) {
-          //   if (detectErrors) {
+        return callback(errorMessage(e));
-          //     self._log('info','writing file '+ ('.embark/stats.json').bold.dim);
+      }
-          //   }
-          //   fs.writeFile(
-          //     path.join(fs.dappPath('.embark'), 'stats.json'),
-          //     JSON.stringify(stats.toJson(webpackOptions.stats)),
-          //     next
-          //   );
-          // }
-          // note: to visualize the stats info in a browser, do...
-          // `npx webpack-bundle-analyzer <dapp_dir>/.embark/stats.json`
-        ], (err) => {
-          callback(err);
-        });
-      });
     });
   }
 }
@@ -214,7 +110,7 @@ process.on('message', (msg) => {
   }
 
   if (msg.action === constants.pipeline.build) {
-    return webpackProcess.build(msg.file, msg.importsList, (err) => {
+    return webpackProcess.build(msg.assets, msg.importsList, (err) => {
       process.send({result: constants.pipeline.built, error: err});
     });
   }

lib/utils/utils.js

@@ -200,10 +200,6 @@ function proposeAlternative(word, _dictionary, _exceptions) {
   return propose(word, dictionary, {threshold: 0.3});
 }
 
-function pwd() {
-  return process.env.PWD || process.cwd();
-}
-
 function getExternalContractUrl(file) {
   const constants = require('../constants');
   let url;
@@ -462,6 +458,15 @@ function interceptLogs(consoleContext, logger) {
   };
 }
 
+function errorMessage(e) {
+  if (typeof e === 'string') {
+    return e;
+  } else if (e && e.message) {
+    return e.message;
+  }
+  return e;
+}
+
 module.exports = {
   joinPath,
   dirname,
@@ -489,7 +494,6 @@ module.exports = {
   extractTar,
   extractZip,
   proposeAlternative,
-  pwd: pwd,
   getExternalContractUrl,
   toChecksumAddress,
   sha3,
@@ -504,5 +508,6 @@ module.exports = {
   groupBy,
   sample,
   last,
-  interceptLogs
+  interceptLogs,
+  errorMessage
 };

package-lock.json

@@ -1313,66 +1313,16 @@
         "util.promisify": "^1.0.0"
       }
     },
-    "babel-plugin-webpack-aliases": {
+    "babel-plugin-module-resolver": {
-      "version": "1.1.3",
+      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-webpack-aliases/-/babel-plugin-webpack-aliases-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/babel-plugin-module-resolver/-/babel-plugin-module-resolver-3.1.1.tgz",
-      "integrity": "sha1-+7oor/c+SDc4949wRMGDrSF5toA=",
+      "integrity": "sha512-1Q77Al4ydp6nYApJ7sQ2fmgz30WuQgJZegIYuyOdbdpxenB/bSezQ3hDPsumIXGlUS4vUIv+EwFjzzXZNWtARw==",
       "requires": {
-        "babel-types": "^6.5.2",
+        "find-babel-config": "^1.1.0",
-        "find-up": "1.1.2"
+        "glob": "^7.1.2",
-      },
+        "pkg-up": "^2.0.0",
-      "dependencies": {
+        "reselect": "^3.0.1",
-        "find-up": {
+        "resolve": "^1.4.0"
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
-          "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
-          "requires": {
-            "path-exists": "^2.0.0",
-            "pinkie-promise": "^2.0.0"
-          }
-        },
-        "path-exists": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
-          "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=",
-          "requires": {
-            "pinkie-promise": "^2.0.0"
-          }
-        }
-      }
-    },
-    "babel-runtime": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
-      "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
-      "requires": {
-        "core-js": "^2.4.0",
-        "regenerator-runtime": "^0.11.0"
-      },
-      "dependencies": {
-        "regenerator-runtime": {
-          "version": "0.11.1",
-          "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
-          "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="
-        }
-      }
-    },
-    "babel-types": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz",
-      "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=",
-      "requires": {
-        "babel-runtime": "^6.26.0",
-        "esutils": "^2.0.2",
-        "lodash": "^4.17.4",
-        "to-fast-properties": "^1.0.3"
-      },
-      "dependencies": {
-        "to-fast-properties": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
-          "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc="
-        }
       }
     },
     "balanced-match": {
@@ -2162,6 +2112,18 @@
       "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
       "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY="
     },
+    "compression-webpack-plugin": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/compression-webpack-plugin/-/compression-webpack-plugin-1.1.11.tgz",
+      "integrity": "sha512-ZVWKrTQhtOP7rDx3M/koXTnRm/iwcYbuCdV+i4lZfAIe32Mov7vUVM0+8Vpz4q0xH+TBUZxq+rM8nhtkDH50YQ==",
+      "requires": {
+        "cacache": "^10.0.1",
+        "find-cache-dir": "^1.0.0",
+        "neo-async": "^2.5.0",
+        "serialize-javascript": "^1.4.0",
+        "webpack-sources": "^1.0.1"
+      }
+    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3631,6 +3593,15 @@
         }
       }
     },
+    "find-babel-config": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/find-babel-config/-/find-babel-config-1.1.0.tgz",
+      "integrity": "sha1-rMAQQ6Z0n+w0Qpvmtk9ULrtdY1U=",
+      "requires": {
+        "json5": "^0.5.1",
+        "path-exists": "^3.0.0"
+      }
+    },
     "find-cache-dir": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-1.0.0.tgz",
@@ -5595,6 +5566,11 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY="
     },
+    "lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
+    },
     "lodash.debounce": {
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
@@ -7856,6 +7832,11 @@
       "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
+    "reselect": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/reselect/-/reselect-3.0.1.tgz",
+      "integrity": "sha1-79qpjqdFEyTQkrKyFjpqHXqaIUc="
+    },
     "resolve": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz",

package.json

@@ -29,12 +29,13 @@
     "ascii-table": "0.0.9",
     "async": "^2.0.1",
     "babel-loader": "8.0.0-beta.4",
-    "babel-plugin-webpack-aliases": "^1.1.3",
+    "babel-plugin-module-resolver": "^3.1.1",
     "bip39": "^2.5.0",
     "chokidar": "^2.0.3",
     "clone-deep": "^4.0.0",
     "colors": "^1.1.2",
     "commander": "^2.15.1",
+    "compression-webpack-plugin": "^1.1.11",
     "css-loader": "^0.28.11",
     "decompress": "^4.2.0",
     "deep-equal": "^1.0.1",
@@ -57,6 +58,7 @@
     "ipfs-api": "17.2.4",
     "is-valid-domain": "0.0.5",
     "live-plugin-manager-git-fix": "^0.12.1",
+    "lodash.clonedeep": "^4.5.0",
     "merge": "^1.2.0",
     "mocha": "^5.2.0",
     "multihashes": "^0.4.13",

webpack.config.js

@@ -1 +0,0 @@
-{}