Merge pull request #403 from embark-framework/features/file-per-process

Process files in multiple separate processes
This commit is contained in:
Iuri Matias 2018-05-10 14:47:22 -04:00 committed by GitHub
commit d84d064ede
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1993 additions and 1634 deletions

View File

@ -8,10 +8,8 @@ const CodeGenerator = require('../contracts/code_generator.js');
const ServicesMonitor = require('./services_monitor.js');
const Watch = require('../pipeline/watch.js');
const LibraryManager = require('../versions/library_manager.js');
const utils = require('../utils/utils');
const constants = require('../constants');
const Pipeline = require('../pipeline/pipeline.js');
const async = require('async');
const fs = require('../core/fs.js');
class Engine {
constructor(options) {
@ -134,81 +132,34 @@ class Engine {
}
pipelineService(_options) {
let self = this;
self.events.emit("status", "Building Assets");
const pipelineProcess = require('child_process').fork(utils.joinPath(__dirname, '../pipeline/pipeline.js'));
async.waterfall([
function getWeb3Version(next) {
self.events.request("version:get:web3", function(web3Version) {
next(null, web3Version);
});
},
function getWeb3Location(web3Version, next) {
if (web3Version === "1.0.0-beta") {
return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
} else {
self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
return next(err, fs.dappPath(location));
});
}
},
function getProviderCode(web3Location, next) {
self.events.once('code-generator-ready', function () {
self.events.request('provider-code', function(providerCode) {
next(null, web3Location, providerCode);
});
});
}
], (err, web3Location, providerCode) => {
if (err) {
return self.logger.error(err);
}
// Setup Pipeline
pipelineProcess.send({action: constants.pipeline.init, options: {
buildDir: self.config.buildDir,
contractsFiles: self.config.contractsFiles,
assetFiles: self.config.assetFiles,
events: self.events,
pipelinePlugins: self.plugins.getPluginsFor('pipeline'),
pluginImports: self.plugins.getPluginsProperty('imports', 'imports'),
web3Location,
providerCode
}});
pipelineProcess.on('message', function (msg) {
if (msg.result === constants.pipeline.built) {
if (self.watch) {
self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched
}
if (msg.error) {
return self.logger.error(msg.error);
}
return self.events.emit('outputDone');
}
if (msg.result === constants.pipeline.log) {
if (self.logger[msg.type]) {
return self.logger[msg.type](self.normalizeInput(msg.message));
}
self.logger.debug(self.normalizeInput(msg.message));
}
const self = this;
this.events.emit("status", "Building Assets");
const pipeline = new Pipeline({
buildDir: this.config.buildDir,
contractsFiles: this.config.contractsFiles,
assetFiles: this.config.assetFiles,
events: this.events,
logger: this.logger,
normalizeInput: this.normalizeInput,
plugins: this.plugins
});
function build() {
const queue = async.queue((task, callback) => {
pipeline.build(task.abi, task.contractsJSON, task.path, callback);
}, 1);
this.events.on('code-generator-ready', function () {
self.events.request('code', function (abi, contractsJSON) {
self.currentAbi = abi;
self.contractsJSON = contractsJSON;
pipelineProcess.send({action: constants.pipeline.build, abi, contractsJSON, path: null});
});
queue.push({abi, contractsJSON, path: null}, () => {
if (self.watch) {
self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched
}
self.events.on('code-generator-ready', function () {
build();
self.events.emit('outputDone');
});
});
build();
});
}

View File

@ -1,54 +1,25 @@
const fs = require('../core/fs.js');
const async = require('async');
const child_process = require('child_process');
const utils = require('../utils/utils.js');
const webpack = require("webpack");
const constants = require('../constants');
const File = require('../core/file');
require("babel-preset-react");
require("babel-preset-es2015");
require("babel-preset-es2016");
require("babel-preset-es2017");
let pipeline;
// Override process.chdir so that we have a partial-implementation PWD for Windows
const realChdir = process.chdir;
process.chdir = (...args) => {
if (!process.env.PWD) {
process.env.PWD = process.cwd();
}
realChdir(...args);
};
class Pipeline {
constructor(options) {
this.buildDir = options.buildDir;
this.contractsFiles = options.contractsFiles;
this.assetFiles = options.assetFiles;
this.pipelinePlugins = options.pipelinePlugins;
this.pluginImports = options.pluginImports;
this.web3Location = options.web3Location;
this.providerCode = options.providerCode;
this.interceptLogs();
}
interceptLogs() {
const context = {};
context.console = console;
context.console.log = this.log.bind(this, 'log');
context.console.warn = this.log.bind(this, 'warn');
context.console.info = this.log.bind(this, 'info');
context.console.debug = this.log.bind(this, 'debug');
context.console.trace = this.log.bind(this, 'trace');
context.console.dir = this.log.bind(this, 'dir');
}
log(type, ...messages) {
process.send({result: constants.pipeline.log, message: messages, type});
this.events = options.events;
this.logger = options.logger;
this.normalizeInput = options.normalizeInput;
this.plugins = options.plugins;
this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
}
build(abi, contractsJSON, path, callback) {
@ -66,7 +37,7 @@ class Pipeline {
importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js');
importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js');
self.pluginImports.forEach(function (importObject) {
self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) {
let [importName, importLocation] = importObject;
importsList[importName] = importLocation;
});
@ -83,14 +54,12 @@ class Pipeline {
}, next);
},
function assetFileWrite(next) {
// limit:1 due to issues when downloading required files such as web3.js
async.eachOfLimit(self.assetFiles, 1, function (files, targetFile, cb) {
// limit:1 due to issues when downloading required files such as web3.js
async.mapLimit(files, 1,
async.eachOf(self.assetFiles, function (files, targetFile, cb) {
async.map(files,
function (file, fileCb) {
file = new File(file); // Re-instantiate a File as through the process, we lose its prototype
console.trace("reading " + file.filename);
self.logger.trace("reading " + file.filename);
// Not a JS file
if (file.filename.indexOf('.js') < 0) {
return file.content(function (fileContent) {
self.runPlugins(file, fileContent, fileCb);
@ -98,36 +67,24 @@ class Pipeline {
}
// JS files
let realCwd;
async.waterfall([
function findImports(next) {
self.webpackRun(file.filename, {}, false, importsList, false, next);
},
function changeCwd(next) {
realCwd = utils.pwd();
process.chdir(fs.embarkPath(''));
next();
},
function runWebpack(next) {
self.webpackRun(file.filename, {}, true, importsList, true, next);
},
const webpackProcess = child_process.fork(utils.joinPath(__dirname, 'webpackProcess.js'));
webpackProcess.send({action: constants.pipeline.init, options: {}});
webpackProcess.send({action: constants.pipeline.build, file, importsList});
function changeCwdBack(next) {
process.chdir(realCwd);
next();
},
function checkFile(next) {
fs.access('./.embark/' + file.filename, (err) => {
if (err) {
console.error("couldn't find file: " + file.filename);
return next("couldn't find file: " + file.filename);
webpackProcess.on('message', function (msg) {
if (msg.result === constants.pipeline.built) {
webpackProcess.disconnect();
return next(msg.error);
}
if (msg.result === constants.pipeline.log) {
if (self.logger[msg.type]) {
return self.logger[msg.type](self.normalizeInput(msg.message));
}
self.logger.debug(self.normalizeInput(msg.message));
}
next();
});
},
@ -146,8 +103,7 @@ class Pipeline {
], function (err, contentFile) {
if (err) {
process.chdir(realCwd);
console.error(err);
self.logger.error(err);
return fileCb(err);
}
@ -156,10 +112,10 @@ class Pipeline {
},
function (err, contentFiles) {
if (err) {
console.error('errors found while generating ' + targetFile);
self.logger.error('errors found while generating ' + targetFile);
}
let dir = targetFile.split('/').slice(0, -1).join('/');
console.trace("creating dir " + self.buildDir + dir);
self.logger.trace("creating dir " + self.buildDir + dir);
fs.mkdirpSync(self.buildDir + dir);
// if it's a directory
@ -172,7 +128,7 @@ class Pipeline {
async.each(contentFiles, function (file, mapCb) {
let filename = file.filename.replace(file.basedir + '/', '');
console.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb);
}, cb);
@ -186,7 +142,7 @@ class Pipeline {
return file.content;
}).join("\n");
console.info("writing file " + (self.buildDir + targetFile).bold.dim);
self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
fs.writeFile(self.buildDir + targetFile, content, cb);
}
);
@ -199,16 +155,10 @@ class Pipeline {
runPlugins(file, fileContent, fileCb) {
const self = this;
if (self.pipelinePlugins.length <= 0) {
return fileCb(null, {
content: fileContent,
filename: file.filename,
path: file.path,
basedir: file.basedir,
modified: true
});
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
}
async.eachSeries(self.pipelinePlugins,
function (plugin, pluginCB) {
function(plugin, pluginCB) {
if (file.options && file.options.skipPipeline) {
return pluginCB();
}
@ -219,85 +169,13 @@ class Pipeline {
},
function (err) {
if (err) {
console.error(err.message);
self.logger.error(err.message);
}
return fileCb(null, {
content: fileContent,
filename: file.filename,
path: file.path,
basedir: file.basedir,
modified: true
});
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
}
);
}
webpackRun(filename, options, includeModules, importsList, detectErrors, callback) {
let defaultOptions = {
entry: fs.dappPath(filename),
output: {
libraryTarget: 'umd',
path: fs.dappPath('.embark'),
filename: filename
},
resolve: {
alias: importsList,
modules: [
fs.embarkPath('node_modules'),
fs.dappPath('node_modules')
]
},
externals: function (context, request, callback) {
callback();
}
};
let webpackOptions = utils.recursiveMerge(defaultOptions, options);
if (includeModules) {
webpackOptions.module = {
rules: [
{
test: /\.css$/,
use: [{loader: "style-loader"}, {loader: "css-loader"}]
},
{
test: /\.scss$/,
use: [{loader: "style-loader"}, {loader: "css-loader"}]
},
{
test: /\.(png|woff|woff2|eot|ttf|svg)$/,
loader: 'url-loader?limit=100000'
},
{
test: /\.js$/,
loader: "babel-loader",
exclude: /(node_modules|bower_components)/,
options: {
presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve),
plugins: ["babel-plugin-webpack-aliases"].map(require.resolve),
compact: false
}
}
]
};
}
webpack(webpackOptions).run((err, stats) => {
if (err) {
console.error(err);
}
if (!detectErrors) {
return callback();
}
if (stats.hasErrors()) {
return callback(stats.toJson().errors.join("\n"));
}
callback();
});
}
buildContracts(contractsJSON, callback) {
fs.mkdirp(fs.dappPath(this.buildDir, 'contracts'), (err) => {
if (err) {
@ -337,21 +215,33 @@ class Pipeline {
let code = "";
async.waterfall([
function getImports(next) {
self.web3Location = self.web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
code += "\nimport Web3 from '" + self.web3Location + "';\n";
function getWeb3Location(next) {
self.events.request("version:get:web3", function(web3Version) {
if (web3Version === "1.0.0-beta") {
return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
} else {
self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
return next(null, fs.dappPath(location));
});
}
});
},
function getImports(web3Location, next) {
web3Location = web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
code += "\nimport Web3 from '" + web3Location + "';\n";
code += "\n if (typeof web3 !== 'undefined') {";
code += "\n } else {";
code += "\n var web3 = new Web3();\n";
code += "\n }";
code += self.providerCode;
self.events.request('provider-code', function(providerCode) {
code += providerCode;
code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n";
code += "\nwindow.web3 = web3;\n";
code += "\nexport default web3;\n";
next();
});
},
function makeDirectory(next) {
fs.mkdirp(fs.dappPath(".embark"), (err, _result) => {
@ -365,19 +255,4 @@ class Pipeline {
}
}
process.on('message', (msg) => {
if (msg.action === constants.pipeline.init) {
pipeline = new Pipeline(msg.options);
return process.send({result: constants.pipeline.initiated});
}
if (msg.action === constants.pipeline.build) {
return pipeline.build(msg.abi, msg.contractsJSON, msg.path, (err) => {
process.send({result: constants.pipeline.built, error: err});
});
}
});
process.on('exit', () => {
process.exit(0);
});
module.exports = Pipeline;

View File

@ -117,7 +117,7 @@ class Watch {
this.logger.trace(files);
let configWatcher = chokidar.watch(files, {
ignored: /[\/\\]\./, persistent: true, ignoreInitial: true, followSymlinks: true
ignored: /[\/\\]\.|tmp_/, persistent: true, ignoreInitial: true, followSymlinks: true
});
this.fileWatchers.push(configWatcher);

View File

@ -0,0 +1,156 @@
const async = require('async');
const webpack = require('webpack');
const utils = require('../utils/utils');
const fs = require('../core/fs');
const constants = require('../constants');
const HardSourceWebpackPlugin = require('hard-source-webpack-plugin');
// Override process.chdir so that we have a partial-implementation PWD for Windows
const realChdir = process.chdir;
process.chdir = (...args) => {
if (!process.env.PWD) {
process.env.PWD = process.cwd();
}
realChdir(...args);
};
let webpackProcess;
class WebpackProcess {
constructor(_options) {
this.interceptLogs();
}
interceptLogs() {
const context = {};
context.console = console;
context.console.log = this.log.bind(this, 'log');
context.console.warn = this.log.bind(this, 'warn');
context.console.info = this.log.bind(this, 'info');
context.console.debug = this.log.bind(this, 'debug');
context.console.trace = this.log.bind(this, 'trace');
context.console.dir = this.log.bind(this, 'dir');
}
log(type, ...messages) {
if (messages[0].indexOf('HardSourceWebpackPlugin')) {
return;
}
process.send({result: constants.pipeline.log, message: messages, type});
}
build(file, importsList, callback) {
const self = this;
let realCwd;
async.waterfall([
function findImports(next) {
self.webpackRun(file.filename, {}, false, importsList, false, next);
},
function changeCwd(next) {
realCwd = utils.pwd();
process.chdir(fs.embarkPath(''));
next();
},
function runWebpack(next) {
self.webpackRun(file.filename, {}, true, importsList, true, next);
},
function changeCwdBack(next) {
process.chdir(realCwd);
next();
}
], (err) => {
process.chdir(realCwd);
callback(err);
});
}
webpackRun(filename, options, includeModules, importsList, detectErrors, callback) {
let defaultOptions = {
entry: fs.dappPath(filename),
output: {
libraryTarget: 'umd',
path: fs.dappPath('.embark'),
filename: filename
},
resolve: {
alias: importsList,
modules: [
fs.embarkPath('node_modules'),
fs.dappPath('node_modules')
]
},
externals: function (context, request, callback) {
callback();
},
plugins: [new HardSourceWebpackPlugin()]
};
let webpackOptions = utils.recursiveMerge(defaultOptions, options);
if (includeModules) {
webpackOptions.module = {
rules: [
{
test: /\.css$/,
use: [{loader: "style-loader"}, {loader: "css-loader"}]
},
{
test: /\.scss$/,
use: [{loader: "style-loader"}, {loader: "css-loader"}]
},
{
test: /\.(png|woff|woff2|eot|ttf|svg)$/,
loader: 'url-loader?limit=100000'
},
{
test: /\.js$/,
loader: "babel-loader",
exclude: /(node_modules|bower_components)/,
options: {
presets: ['babel-preset-es2016', 'babel-preset-es2017', 'babel-preset-react'].map(require.resolve),
plugins: ["babel-plugin-webpack-aliases"].map(require.resolve),
compact: false
}
}
]
};
}
webpack(webpackOptions).run((err, stats) => {
if (err) {
console.error(err);
}
if (!detectErrors) {
return callback();
}
if (stats.hasErrors()) {
return callback(stats.toJson().errors.join("\n"));
}
callback();
});
}
}
process.on('message', (msg) => {
if (msg.action === constants.pipeline.init) {
webpackProcess = new WebpackProcess(msg.options);
return process.send({result: constants.pipeline.initiated});
}
if (msg.action === constants.pipeline.build) {
return webpackProcess.build(msg.file, msg.importsList, (err) => {
process.send({result: constants.pipeline.built, error: err});
});
}
});
process.on('exit', () => {
process.exit(0);
});

3149
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -44,11 +44,11 @@
"follow-redirects": "^1.2.4",
"fs-extra": "^2.0.0",
"globule": "^1.1.0",
"hard-source-webpack-plugin": "^0.6.6",
"http-shutdown": "^1.2.0",
"ipfs-api": "17.2.4",
"live-plugin-manager": "https://github.com/iurimatias/live-plugin-manager.git",
"merge": "^1.2.0",
"mocha": "^2.2.5",
"orbit-db": "^0.17.3",
"parse-json": "^4.0.0",
"promptly": "^2.1.0",