From 91e5e9c99099e8bfa3a9fb0ca73a94c323933d8b Mon Sep 17 00:00:00 2001 From: emizzle Date: Thu, 11 Oct 2018 22:19:27 +1100 Subject: [PATCH] Do not run webpack for file changes that do not need webpack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For file changes that do not require a webpack run, ie HTML, the assets will still be copied to the output directory, but webpack will not run (as it’s too slow). --- lib/core/engine.js | 22 ++--- lib/modules/pipeline/index.js | 26 +++++- lib/modules/pipeline/webpackConfigReader.js | 56 ++++++++++++ lib/modules/pipeline/webpackProcess.js | 94 ++++++++------------- lib/modules/watcher/index.js | 8 +- 5 files changed, 128 insertions(+), 78 deletions(-) create mode 100644 lib/modules/pipeline/webpackConfigReader.js diff --git a/lib/core/engine.js b/lib/core/engine.js index de4bf8d8..53c3d064 100644 --- a/lib/core/engine.js +++ b/lib/core/engine.js @@ -108,9 +108,9 @@ class Engine { this.registerModule('pipeline', { webpackConfigName: this.webpackConfigName }); - this.events.on('code-generator-ready', function () { + this.events.on('code-generator-ready', function (modifiedAsset) { self.events.request('code', function (abi, contractsJSON) { - self.events.request('pipeline:build', {abi, contractsJSON}, () => { + self.events.request('pipeline:build', {abi, contractsJSON, modifiedAsset}, () => { self.events.emit('outputDone'); }); }); @@ -162,17 +162,17 @@ class Engine { this.registerModule('code_generator', {plugins: self.plugins, env: self.env}); - const generateCode = function () { + const generateCode = function (modifiedAsset) { self.events.request("code-generator:embarkjs:build", () => { - self.events.emit('code-generator-ready'); + self.events.emit('code-generator-ready', modifiedAsset); }); }; - const cargo = async.cargo((_tasks, callback) => { - generateCode(); + const cargo = async.queue((task, callback) => { + generateCode(task.modifiedAsset); self.events.once('outputDone', callback); - }); - const addToCargo = function () { - cargo.push({}); + }, 10); + const addToCargo = function (modifiedAsset) { + cargo.push({modifiedAsset}); }; this.events.on('contractsDeployed', addToCargo); @@ -198,7 +198,7 @@ class Engine { this.registerModule('console_listener', {ipc: self.ipc}); this.registerModule('deployment', {plugins: this.plugins, onlyCompile: options.onlyCompile}); - this.events.on('file-event', function (fileType) { + this.events.on('file-event', function ({fileType, path}) { clearTimeout(self.fileTimeout); self.fileTimeout = setTimeout(() => { // TODO: still need to redeploy contracts because the original contracts @@ -206,7 +206,7 @@ class Engine { self.config.reloadConfig(); if (fileType === 'asset') { // Throttle file changes so we re-write only once for all files - self.events.emit('asset-changed', self.contractsManager); + self.events.emit('asset-changed', path); } // TODO: for now need to deploy on asset changes as well // because the contractsManager config is corrupted after a deploy diff --git a/lib/modules/pipeline/index.js b/lib/modules/pipeline/index.js index b82516ab..be0c818e 100644 --- a/lib/modules/pipeline/index.js +++ b/lib/modules/pipeline/index.js @@ -3,6 +3,7 @@ const async = require('async'); const utils = require('../../utils/utils.js'); const ProcessLauncher = require('../../core/processes/processLauncher'); const constants = require('../../constants'); +const WebpackConfigReader = require('../pipeline/webpackConfigReader'); class Pipeline { constructor(embark, options) { @@ -18,11 +19,11 @@ class Pipeline { this.pipelineConfig = embark.config.pipelineConfig; this.isFirstBuild = true; - this.events.setCommandHandler('pipeline:build', (options, callback) => this.build(callback)); + this.events.setCommandHandler('pipeline:build', (options, callback) => this.build(options, callback)); fs.removeSync(this.buildDir); } - build(callback) { + build({modifiedAsset}, callback) { let self = this; const importsList = {}; let placeholderPage; @@ -82,7 +83,26 @@ class Pipeline { }); }); }, - function runWebpack(next) { + function shouldRunWebpack(next){ + // assuming we got here because an asset was changed, let's check our webpack config + // to see if the changed asset requires webpack to run + if(modifiedAsset){ + const configReader = new WebpackConfigReader({webpackConfigName: self.webpackConfigName}); + return configReader.readConfig((err, config) => { + if(err) return next(err); + + if (typeof config !== 'object' || config === null) { + return next(__('bad webpack config, the resolved config was null or not an object')); + } + + const shouldRun = config.module.rules.some(rule => rule.test.test(modifiedAsset)); + return next(null, !shouldRun); + }); + } + next(null, false); + }, + function runWebpack(shouldNotRun, next) { + if(shouldNotRun) return next(); self.logger.info(__(`running webpack with '${self.webpackConfigName}' config...`)); const assets = Object.keys(self.assetFiles).filter(key => key.match(/\.js$/)); if (!assets || !assets.length) { diff --git a/lib/modules/pipeline/webpackConfigReader.js b/lib/modules/pipeline/webpackConfigReader.js new file mode 100644 index 00000000..ce0a1d72 --- /dev/null +++ b/lib/modules/pipeline/webpackConfigReader.js @@ -0,0 +1,56 @@ + +const {errorMessage} = require('../../utils/utils'); +const fs = require('../../core/fs'); + +class WebpackConfigReader { + constructor(options) { + this.webpackConfigName = options.webpackConfigName; + } + + async readConfig(callback){ + const dappConfigPath = fs.dappPath('webpack.config.js'); + const defaultConfigPath = fs.embarkPath('lib/modules/pipeline', 'webpack.config.js'); + + let config, configPath; + try { + if (fs.existsSync(dappConfigPath)) { + configPath = dappConfigPath; + delete require.cache[configPath]; + } else { + configPath = defaultConfigPath; + } + config = require(configPath); + // valid config types: https://webpack.js.org/configuration/configuration-types/ + // + function that returns a config object + // + function that returns a promise for a config object + // + array of named config objects + // + config object + if (typeof config === 'function') { + // if(Promise.resolve(config)){ + // return config(this.webpackConfigName).then(config => { + // callback(null, config); + // }); + // } + + config = await config(this.webpackConfigName); + //return callback(null, config); + } else if (Array.isArray(config)) { + config = config.filter(cfg => cfg.name === this.webpackConfigName); + if (!config.length) { + return callback(`no webpack config has the name '${this.webpackConfigName}'`); + } + if (config.length > 1) { + console.warn(`detected ${config.length} webpack configs having the name '${this.webpackConfigName}', using the first one`); + } + config = config[0]; + //return callback(null, config); + } + callback(null, config); + } catch (e) { + console.error(`error while loading webpack config ${configPath}`); + callback(errorMessage(e)); + } + } +} + +module.exports = WebpackConfigReader; diff --git a/lib/modules/pipeline/webpackProcess.js b/lib/modules/pipeline/webpackProcess.js index 8cd8932c..83a16e9a 100644 --- a/lib/modules/pipeline/webpackProcess.js +++ b/lib/modules/pipeline/webpackProcess.js @@ -4,6 +4,7 @@ const ProcessWrapper = require('../../core/processes/processWrapper'); const webpack = require('webpack'); const writeFile = require('util').promisify(require('fs').writeFile); const {errorMessage} = require('../../utils/utils'); +const WebpackConfigReader = require('./webpackConfigReader'); let webpackProcess; @@ -40,70 +41,43 @@ class WebpackProcess extends ProcessWrapper { return callback(errorMessage(e)); } - const dappConfigPath = fs.dappPath('webpack.config.js'); - const defaultConfigPath = fs.embarkPath('lib/modules/pipeline', 'webpack.config.js'); - - let config, configPath; - try { - if (fs.existsSync(dappConfigPath)) { - configPath = dappConfigPath; - delete require.cache[configPath]; - } else { - configPath = defaultConfigPath; - } - config = require(configPath); - // valid config types: https://webpack.js.org/configuration/configuration-types/ - // + function that returns a config object - // + function that returns a promise for a config object - // + array of named config objects - // + config object - if (typeof config === 'function') { - config = await config(this.webpackConfigName); - } else if (Array.isArray(config)) { - config = config.filter(cfg => cfg.name === this.webpackConfigName); - if (!config.length) { - return callback(`no webpack config has the name '${this.webpackConfigName}'`); - } - if (config.length > 1) { - console.warn(`detected ${config.length} webpack configs having the name '${this.webpackConfigName}', using the first one`); - } - config = config[0]; - } - } catch (e) { - console.error(`error while loading webpack config ${configPath}`); - callback(errorMessage(e)); - } - - if (typeof config !== 'object' || config === null) { - return callback('bad webpack config, the resolved config was null or not an object'); - } - - webpack(config).run(async (err, stats) => { + const configReader = new WebpackConfigReader({webpackConfigName: this.webpackConfigName}); + configReader.readConfig((err, config) => { if (err) { - return callback(errorMessage(err)); + return callback(err); } - if (!config.stats || config.stats === 'none') { - return callback(); + + if (typeof config !== 'object' || config === null) { + return callback('bad webpack config, the resolved config was null or not an object'); } - try { - this._log('info', 'writing file '+ ('.embark/stats.report').bold.dim); - await writeFile( - fs.dappPath('.embark/stats.report'), - stats.toString(config.stats) - ); - this._log('info', 'writing file ' + ('.embark/stats.json').bold.dim); - await writeFile( - fs.dappPath('.embark/stats.json'), - JSON.stringify(stats.toJson(config.stats)) - ); - if (stats.hasErrors()) { - const errors = stats.toJson(config.stats).errors.join('\n'); - return callback(errors); + + webpack(config).run(async (err, stats) => { + if (err) { + return callback(errorMessage(err)); } - callback(); - } catch (e) { - return callback(errorMessage(e)); - } + if (!config.stats || config.stats === 'none') { + return callback(); + } + try { + this._log('info', 'writing file ' + ('.embark/stats.report').bold.dim); + await writeFile( + fs.dappPath('.embark/stats.report'), + stats.toString(config.stats) + ); + this._log('info', 'writing file ' + ('.embark/stats.json').bold.dim); + await writeFile( + fs.dappPath('.embark/stats.json'), + JSON.stringify(stats.toJson(config.stats)) + ); + if (stats.hasErrors()) { + const errors = stats.toJson(config.stats).errors.join('\n'); + return callback(errors); + } + callback(); + } catch (e) { + return callback(errorMessage(e)); + } + }); }); } } diff --git a/lib/modules/watcher/index.js b/lib/modules/watcher/index.js index 26dc1ee4..e6c20f40 100644 --- a/lib/modules/watcher/index.js +++ b/lib/modules/watcher/index.js @@ -89,7 +89,7 @@ class Watcher { function (eventName, path) { self.logger.info(`${eventName}: ${path}`); self.events.emit('file-' + eventName, 'asset', path); - self.events.emit('file-event', 'asset', path); + self.events.emit('file-event', {fileType: 'asset', path}); }, function () { callback(); @@ -104,7 +104,7 @@ class Watcher { function (eventName, path) { self.logger.info(`${eventName}: ${path}`); self.events.emit('file-' + eventName, 'contract', path); - self.events.emit('file-event', 'contract', path); + self.events.emit('file-event', {fileType: 'contract', path}); }, function () { callback(); @@ -154,7 +154,7 @@ class Watcher { function (eventName, path) { self.logger.info(`${eventName}: ${path}`); self.events.emit('file-' + eventName, 'config', path); - self.events.emit('file-event', 'config', path); + self.events.emit('file-event', {fileType: 'config', path}); }, function () { callback(); @@ -177,7 +177,7 @@ class Watcher { this.watchFiles(filesToWatch, (eventName, path) => { this.logger.info(`${eventName}: ${path}`); this.events.emit('file-' + eventName, 'config', path); - this.events.emit('file-event', 'config', path); + this.events.emit('file-event', {fileType: 'config', path}); }, callback); }