From ed1d0ce67c55dac094ef2d65711b7c729478e27c Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 10 May 2018 10:10:09 -0400 Subject: [PATCH] make only webpack in a separate process --- lib/core/engine.js | 91 +++---------- lib/pipeline/pipeline.js | 230 +++++++++------------------------ lib/pipeline/webpackProcess.js | 151 ++++++++++++++++++++++ 3 files changed, 228 insertions(+), 244 deletions(-) create mode 100644 lib/pipeline/webpackProcess.js diff --git a/lib/core/engine.js b/lib/core/engine.js index 82789206e..de1d09f14 100644 --- a/lib/core/engine.js +++ b/lib/core/engine.js @@ -8,10 +8,7 @@ const CodeGenerator = require('../contracts/code_generator.js'); const ServicesMonitor = require('./services_monitor.js'); const Watch = require('../pipeline/watch.js'); const LibraryManager = require('../versions/library_manager.js'); -const utils = require('../utils/utils'); -const constants = require('../constants'); -const async = require('async'); -const fs = require('../core/fs.js'); +let Pipeline = require('../pipeline/pipeline.js'); class Engine { constructor(options) { @@ -134,81 +131,27 @@ class Engine { } pipelineService(_options) { - let self = this; - self.events.emit("status", "Building Assets"); - const pipelineProcess = require('child_process').fork(utils.joinPath(__dirname, '../pipeline/pipeline.js')); - - async.waterfall([ - function getWeb3Version(next) { - self.events.request("version:get:web3", function(web3Version) { - next(null, web3Version); - }); - }, - function getWeb3Location(web3Version, next) { - if (web3Version === "1.0.0-beta") { - return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js"))); - } else { - self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) { - return next(err, fs.dappPath(location)); - }); - } - }, - function getProviderCode(web3Location, next) { - self.events.once('code-generator-ready', function () { - self.events.request('provider-code', function(providerCode) { - next(null, web3Location, providerCode); - }); - }); - } - ], (err, web3Location, providerCode) => { - if (err) { - return self.logger.error(err); - } - - // Setup Pipeline - pipelineProcess.send({action: constants.pipeline.init, options: { - buildDir: self.config.buildDir, - contractsFiles: self.config.contractsFiles, - assetFiles: self.config.assetFiles, - events: self.events, - pipelinePlugins: self.plugins.getPluginsFor('pipeline'), - pluginImports: self.plugins.getPluginsProperty('imports', 'imports'), - web3Location, - providerCode - }}); - - pipelineProcess.on('message', function (msg) { - if (msg.result === constants.pipeline.built) { + const self = this; + this.events.emit("status", "Building Assets"); + const pipeline = new Pipeline({ + buildDir: this.config.buildDir, + contractsFiles: this.config.contractsFiles, + assetFiles: this.config.assetFiles, + events: this.events, + logger: this.logger, + plugins: this.plugins + }); + this.events.on('code-generator-ready', function () { + self.events.request('code', function (abi, contractsJSON) { + self.currentAbi = abi; + self.contractsJSON = contractsJSON; + pipeline.build(abi, contractsJSON, null, function() { if (self.watch) { self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched } - if (msg.error) { - return self.logger.error(msg.error); - } - return self.events.emit('outputDone'); - } - - if (msg.result === constants.pipeline.log) { - if (self.logger[msg.type]) { - return self.logger[msg.type](self.normalizeInput(msg.message)); - } - self.logger.debug(self.normalizeInput(msg.message)); - } - }); - - function build() { - self.events.request('code', function (abi, contractsJSON) { - self.currentAbi = abi; - self.contractsJSON = contractsJSON; - pipelineProcess.send({action: constants.pipeline.build, abi, contractsJSON, path: null}); + self.events.emit('outputDone'); }); - } - - self.events.on('code-generator-ready', function () { - build(); }); - - build(); }); } diff --git a/lib/pipeline/pipeline.js b/lib/pipeline/pipeline.js index dace18d60..f830985dc 100644 --- a/lib/pipeline/pipeline.js +++ b/lib/pipeline/pipeline.js @@ -1,7 +1,7 @@ const fs = require('../core/fs.js'); const async = require('async'); +const child_process = require('child_process'); const utils = require('../utils/utils.js'); -const webpack = require("webpack"); const constants = require('../constants'); const File = require('../core/file'); @@ -10,45 +10,15 @@ require("babel-preset-es2015"); require("babel-preset-es2016"); require("babel-preset-es2017"); -let pipeline; - -// Override process.chdir so that we have a partial-implementation PWD for Windows -const realChdir = process.chdir; -process.chdir = (...args) => { - if (!process.env.PWD) { - process.env.PWD = process.cwd(); - } - realChdir(...args); -}; - class Pipeline { constructor(options) { this.buildDir = options.buildDir; this.contractsFiles = options.contractsFiles; this.assetFiles = options.assetFiles; - this.pipelinePlugins = options.pipelinePlugins; - this.pluginImports = options.pluginImports; - this.web3Location = options.web3Location; - this.providerCode = options.providerCode; - - this.interceptLogs(); - } - - interceptLogs() { - const context = {}; - context.console = console; - - context.console.log = this.log.bind(this, 'log'); - context.console.warn = this.log.bind(this, 'warn'); - context.console.info = this.log.bind(this, 'info'); - context.console.debug = this.log.bind(this, 'debug'); - context.console.trace = this.log.bind(this, 'trace'); - context.console.dir = this.log.bind(this, 'dir'); - } - - log(type, ...messages) { - process.send({result: constants.pipeline.log, message: messages, type}); + this.events = options.events; + this.logger = options.logger; + this.plugins = options.plugins; } build(abi, contractsJSON, path, callback) { @@ -66,7 +36,7 @@ class Pipeline { importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js'); importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js'); - self.pluginImports.forEach(function (importObject) { + self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) { let [importName, importLocation] = importObject; importsList[importName] = importLocation; }); @@ -89,7 +59,7 @@ class Pipeline { async.mapLimit(files, 1, function (file, fileCb) { file = new File(file); // Re-instantiate a File as through the process, we lose its prototype - console.trace("reading " + file.filename); + self.logger.trace("reading " + file.filename); if (file.filename.indexOf('.js') < 0) { return file.content(function (fileContent) { @@ -98,33 +68,34 @@ class Pipeline { } // JS files - let realCwd; - async.waterfall([ - - function findImports(next) { - self.webpackRun(file.filename, {}, false, importsList, false, next); - }, - - function changeCwd(next) { - realCwd = utils.pwd(); - process.chdir(fs.embarkPath('')); - next(); - }, - function runWebpack(next) { - self.webpackRun(file.filename, {}, true, importsList, true, next); - }, + const webpackProcess = child_process.fork(utils.joinPath(__dirname, 'webpackProcess.js')); + webpackProcess.send({action: constants.pipeline.init, options: {}}); + webpackProcess.send({action: constants.pipeline.build, file, importsList}); - function changeCwdBack(next) { - process.chdir(realCwd); - next(); + webpackProcess.on('message', function (msg) { + if (msg.result === constants.pipeline.built) { + webpackProcess.disconnect(); + if (msg.error) { + return next(msg.error); + } + return next(); + } + + if (msg.result === constants.pipeline.log) { + if (self.logger[msg.type]) { + return self.logger[msg.type](self.normalizeInput(msg.message)); + } + self.logger.debug(self.normalizeInput(msg.message)); + } + }); }, function checkFile(next) { fs.access('./.embark/' + file.filename, (err) => { if (err) { - console.error("couldn't find file: " + file.filename); + self.logger.error("couldn't find file: " + file.filename); return next("couldn't find file: " + file.filename); } next(); @@ -146,8 +117,7 @@ class Pipeline { ], function (err, contentFile) { if (err) { - process.chdir(realCwd); - console.error(err); + self.logger.error(err); return fileCb(err); } @@ -156,10 +126,10 @@ class Pipeline { }, function (err, contentFiles) { if (err) { - console.error('errors found while generating ' + targetFile); + self.logger.error('errors found while generating ' + targetFile); } let dir = targetFile.split('/').slice(0, -1).join('/'); - console.trace("creating dir " + self.buildDir + dir); + self.logger.trace("creating dir " + self.buildDir + dir); fs.mkdirpSync(self.buildDir + dir); // if it's a directory @@ -172,7 +142,7 @@ class Pipeline { async.each(contentFiles, function (file, mapCb) { let filename = file.filename.replace(file.basedir + '/', ''); - console.info("writing file " + (self.buildDir + targetDir + filename).bold.dim); + self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim); fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb); }, cb); @@ -186,7 +156,7 @@ class Pipeline { return file.content; }).join("\n"); - console.info("writing file " + (self.buildDir + targetFile).bold.dim); + self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim); fs.writeFile(self.buildDir + targetFile, content, cb); } ); @@ -198,17 +168,12 @@ class Pipeline { runPlugins(file, fileContent, fileCb) { const self = this; - if (self.pipelinePlugins.length <= 0) { - return fileCb(null, { - content: fileContent, - filename: file.filename, - path: file.path, - basedir: file.basedir, - modified: true - }); + let pipelinePlugins = self.plugins.getPluginsFor('pipeline'); + if (pipelinePlugins.length <= 0) { + return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true}); } - async.eachSeries(self.pipelinePlugins, - function (plugin, pluginCB) { + async.eachSeries(pipelinePlugins, + function(plugin, pluginCB) { if (file.options && file.options.skipPipeline) { return pluginCB(); } @@ -219,85 +184,13 @@ class Pipeline { }, function (err) { if (err) { - console.error(err.message); + self.logger.error(err.message); } - return fileCb(null, { - content: fileContent, - filename: file.filename, - path: file.path, - basedir: file.basedir, - modified: true - }); + return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true}); } ); } - webpackRun(filename, options, includeModules, importsList, detectErrors, callback) { - let defaultOptions = { - entry: fs.dappPath(filename), - output: { - libraryTarget: 'umd', - path: fs.dappPath('.embark'), - filename: filename - }, - resolve: { - alias: importsList, - modules: [ - fs.embarkPath('node_modules'), - fs.dappPath('node_modules') - ] - }, - externals: function (context, request, callback) { - callback(); - } - }; - - let webpackOptions = utils.recursiveMerge(defaultOptions, options); - - if (includeModules) { - webpackOptions.module = { - rules: [ - { - test: /\.css$/, - use: [{loader: "style-loader"}, {loader: "css-loader"}] - }, - { - test: /\.scss$/, - use: [{loader: "style-loader"}, {loader: "css-loader"}] - }, - { - test: /\.(png|woff|woff2|eot|ttf|svg)$/, - loader: 'url-loader?limit=100000' - }, - { - test: /\.js$/, - loader: "babel-loader", - exclude: /(node_modules|bower_components)/, - options: { - presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve), - plugins: ["babel-plugin-webpack-aliases"].map(require.resolve), - compact: false - } - } - ] - }; - } - - webpack(webpackOptions).run((err, stats) => { - if (err) { - console.error(err); - } - if (!detectErrors) { - return callback(); - } - - if (stats.hasErrors()) { - return callback(stats.toJson().errors.join("\n")); - } - callback(); - }); - } - buildContracts(contractsJSON, callback) { fs.mkdirp(fs.dappPath(this.buildDir, 'contracts'), (err) => { if (err) { @@ -337,21 +230,33 @@ class Pipeline { let code = ""; async.waterfall([ - function getImports(next) { - self.web3Location = self.web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes - code += "\nimport Web3 from '" + self.web3Location + "';\n"; + function getWeb3Location(next) { + self.events.request("version:get:web3", function(web3Version) { + if (web3Version === "1.0.0-beta") { + return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js"))); + } else { + self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) { + return next(null, fs.dappPath(location)); + }); + } + }); + }, + function getImports(web3Location, next) { + web3Location = web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes + code += "\nimport Web3 from '" + web3Location + "';\n"; code += "\n if (typeof web3 !== 'undefined') {"; code += "\n } else {"; code += "\n var web3 = new Web3();\n"; code += "\n }"; - code += self.providerCode; - code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n"; - code += "\nwindow.web3 = web3;\n"; - code += "\nexport default web3;\n"; - - next(); + self.events.request('provider-code', function(providerCode) { + code += providerCode; + code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n"; + code += "\nwindow.web3 = web3;\n"; + code += "\nexport default web3;\n"; + next(); + }); }, function makeDirectory(next) { fs.mkdirp(fs.dappPath(".embark"), (err, _result) => { @@ -365,19 +270,4 @@ class Pipeline { } } -process.on('message', (msg) => { - if (msg.action === constants.pipeline.init) { - pipeline = new Pipeline(msg.options); - return process.send({result: constants.pipeline.initiated}); - } - - if (msg.action === constants.pipeline.build) { - return pipeline.build(msg.abi, msg.contractsJSON, msg.path, (err) => { - process.send({result: constants.pipeline.built, error: err}); - }); - } -}); - -process.on('exit', () => { - process.exit(0); -}); +module.exports = Pipeline; diff --git a/lib/pipeline/webpackProcess.js b/lib/pipeline/webpackProcess.js new file mode 100644 index 000000000..5de205327 --- /dev/null +++ b/lib/pipeline/webpackProcess.js @@ -0,0 +1,151 @@ +const async = require('async'); +const webpack = require('webpack'); +const utils = require('../utils/utils'); +const fs = require('../core/fs'); +const constants = require('../constants'); + +// Override process.chdir so that we have a partial-implementation PWD for Windows +const realChdir = process.chdir; +process.chdir = (...args) => { + if (!process.env.PWD) { + process.env.PWD = process.cwd(); + } + realChdir(...args); +}; + +let webpackProcess; + +class WebpackProcess { + constructor(_options) { + this.interceptLogs(); + } + + interceptLogs() { + const context = {}; + context.console = console; + + context.console.log = this.log.bind(this, 'log'); + context.console.warn = this.log.bind(this, 'warn'); + context.console.info = this.log.bind(this, 'info'); + context.console.debug = this.log.bind(this, 'debug'); + context.console.trace = this.log.bind(this, 'trace'); + context.console.dir = this.log.bind(this, 'dir'); + } + + log(type, ...messages) { + process.send({result: constants.pipeline.log, message: messages, type}); + } + + build(file, importsList, callback) { + const self = this; + let realCwd; + + async.waterfall([ + function findImports(next) { + self.webpackRun(file.filename, {}, false, importsList, false, next); + }, + + function changeCwd(next) { + realCwd = utils.pwd(); + process.chdir(fs.embarkPath('')); + next(); + }, + + function runWebpack(next) { + self.webpackRun(file.filename, {}, true, importsList, true, next); + }, + + function changeCwdBack(next) { + process.chdir(realCwd); + next(); + } + ], (err) => { + process.chdir(realCwd); + callback(err); + }); + } + + + webpackRun(filename, options, includeModules, importsList, detectErrors, callback) { + let defaultOptions = { + entry: fs.dappPath(filename), + output: { + libraryTarget: 'umd', + path: fs.dappPath('.embark'), + filename: filename + }, + resolve: { + alias: importsList, + modules: [ + fs.embarkPath('node_modules'), + fs.dappPath('node_modules') + ] + }, + externals: function (context, request, callback) { + callback(); + } + }; + + let webpackOptions = utils.recursiveMerge(defaultOptions, options); + + if (includeModules) { + webpackOptions.module = { + rules: [ + { + test: /\.css$/, + use: [{loader: "style-loader"}, {loader: "css-loader"}] + }, + { + test: /\.scss$/, + use: [{loader: "style-loader"}, {loader: "css-loader"}] + }, + { + test: /\.(png|woff|woff2|eot|ttf|svg)$/, + loader: 'url-loader?limit=100000' + }, + { + test: /\.js$/, + loader: "babel-loader", + exclude: /(node_modules|bower_components)/, + options: { + presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve), + plugins: ["babel-plugin-webpack-aliases"].map(require.resolve), + compact: false + } + } + ] + }; + } + + webpack(webpackOptions).run((err, stats) => { + if (err) { + console.error(err); + } + if (!detectErrors) { + return callback(); + } + + if (stats.hasErrors()) { + return callback(stats.toJson().errors.join("\n")); + } + callback(); + }); + } +} + +process.on('message', (msg) => { + if (msg.action === constants.pipeline.init) { + webpackProcess = new WebpackProcess(msg.options); + return process.send({result: constants.pipeline.initiated}); + } + + if (msg.action === constants.pipeline.build) { + return webpackProcess.build(msg.file, msg.importsList, (err) => { + process.send({result: constants.pipeline.built, error: err}); + }); + } +}); + +process.on('exit', () => { + process.exit(0); +});