Merge pull request #400 from embark-framework/features/write-files-process

Write files in a different process
This commit is contained in:
Iuri Matias 2018-05-09 16:57:43 -04:00 committed by GitHub
commit 911efe4e84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 403 additions and 256 deletions

View File

@ -15,5 +15,12 @@
"events": {
"contractFilesChanged": "contractFilesChanged",
"contractConfigChanged": "contractConfigChanged"
},
"pipeline": {
"init": "init",
"build": "build",
"initiated": "initiated",
"built": "built",
"log": "log"
}
}

View File

@ -1,14 +1,17 @@
let Web3 = require('web3');
let Events = require('./events.js');
let Logger = require('./logger.js');
let Config = require('./config.js');
let ContractsManager = require('../contracts/contracts.js');
let DeployManager = require('../contracts/deploy_manager.js');
let CodeGenerator = require('../contracts/code_generator.js');
let ServicesMonitor = require('./services_monitor.js');
let Pipeline = require('../pipeline/pipeline.js');
let Watch = require('../pipeline/watch.js');
let LibraryManager = require('../versions/library_manager.js');
const Web3 = require('web3');
const Events = require('./events.js');
const Logger = require('./logger.js');
const Config = require('./config.js');
const ContractsManager = require('../contracts/contracts.js');
const DeployManager = require('../contracts/deploy_manager.js');
const CodeGenerator = require('../contracts/code_generator.js');
const ServicesMonitor = require('./services_monitor.js');
const Watch = require('../pipeline/watch.js');
const LibraryManager = require('../versions/library_manager.js');
const utils = require('../utils/utils');
const constants = require('../constants');
const async = require('async');
const fs = require('../core/fs.js');
class Engine {
constructor(options) {
@ -41,12 +44,7 @@ class Engine {
}
}
doInterceptLogs() {
var self = this;
let context = {};
context.console = console;
let normalizeInput = function(input) {
normalizeInput(input) {
let args = Object.values(input);
if (args.length === 0) {
return "";
@ -61,26 +59,31 @@ class Engine {
if (Array.isArray(x)) { return x.join(','); }
return x;
}).toString() + ']');
};
}
doInterceptLogs() {
var self = this;
let context = {};
context.console = console;
context.console.log = function() {
self.logger.info(normalizeInput(arguments));
self.logger.info(self.normalizeInput(arguments));
};
context.console.warn = function() {
self.logger.warn(normalizeInput(arguments));
self.logger.warn(self.normalizeInput(arguments));
};
context.console.info = function() {
self.logger.info(normalizeInput(arguments));
self.logger.info(self.normalizeInput(arguments));
};
context.console.debug = function() {
// TODO: ue JSON.stringify
self.logger.debug(normalizeInput(arguments));
self.logger.debug(self.normalizeInput(arguments));
};
context.console.trace = function() {
self.logger.trace(normalizeInput(arguments));
self.logger.trace(self.normalizeInput(arguments));
};
context.console.dir = function() {
self.logger.dir(normalizeInput(arguments));
self.logger.dir(self.normalizeInput(arguments));
};
}
@ -132,26 +135,80 @@ class Engine {
pipelineService(_options) {
let self = this;
this.events.emit("status", "Building Assets");
let pipeline = new Pipeline({
buildDir: this.config.buildDir,
contractsFiles: this.config.contractsFiles,
assetFiles: this.config.assetFiles,
events: this.events,
logger: this.logger,
plugins: this.plugins
self.events.emit("status", "Building Assets");
const pipelineProcess = require('child_process').fork(utils.joinPath(__dirname, '../pipeline/pipeline.js'));
async.waterfall([
function getWeb3Version(next) {
self.events.request("version:get:web3", function(web3Version) {
next(null, web3Version);
});
this.events.on('code-generator-ready', function () {
self.events.request('code', function (abi, contractsJSON) {
self.currentAbi = abi;
self.contractsJSON = contractsJSON;
pipeline.build(abi, contractsJSON, null, function() {
},
function getWeb3Location(web3Version, next) {
if (web3Version === "1.0.0-beta") {
return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
} else {
self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
return next(err, fs.dappPath(location));
});
}
},
function getProviderCode(web3Location, next) {
self.events.once('code-generator-ready', function () {
self.events.request('provider-code', function(providerCode) {
next(null, web3Location, providerCode);
});
});
}
], (err, web3Location, providerCode) => {
if (err) {
return self.logger.error(err);
}
// Setup Pipeline
pipelineProcess.send({action: constants.pipeline.init, options: {
buildDir: self.config.buildDir,
contractsFiles: self.config.contractsFiles,
assetFiles: self.config.assetFiles,
events: self.events,
pipelinePlugins: self.plugins.getPluginsFor('pipeline'),
pluginImports: self.plugins.getPluginsProperty('imports', 'imports'),
web3Location,
providerCode
}});
pipelineProcess.on('message', function (msg) {
if (msg.result === constants.pipeline.built) {
if (self.watch) {
self.watch.restart(); // Necessary because changing a file while it is writing can stop it from being watched
}
self.events.emit('outputDone');
if (msg.error) {
return self.logger.error(msg.error);
}
return self.events.emit('outputDone');
}
if (msg.result === constants.pipeline.log) {
if (self.logger[msg.type]) {
return self.logger[msg.type](self.normalizeInput(msg.message));
}
self.logger.debug(self.normalizeInput(msg.message));
}
});
function build() {
self.events.request('code', function (abi, contractsJSON) {
self.currentAbi = abi;
self.contractsJSON = contractsJSON;
pipelineProcess.send({action: constants.pipeline.build, abi, contractsJSON, path: null});
});
}
self.events.on('code-generator-ready', function () {
build();
});
build();
});
}

View File

@ -11,6 +11,10 @@ function mkdirp() {
return fs.mkdirp.apply(fs.mkdirp, arguments);
}
function copy() {
return fs.copy.apply(fs.copy, arguments);
}
function copySync() {
return fs.copySync.apply(fs.copySync, arguments);
}
@ -19,6 +23,10 @@ function appendFileSync() {
return fs.appendFileSync.apply(fs.writeFileSync, arguments);
}
function writeFile() {
return fs.writeFile.apply(fs.writeFileSync, arguments);
}
function writeFileSync() {
return fs.writeFileSync.apply(fs.writeFileSync, arguments);
}
@ -45,10 +53,18 @@ function writeJSONSync() {
return fs.writeJSONSync.apply(fs.writeJSONSync, arguments);
}
function writeJson() {
return fs.writeJson.apply(fs.writeJson, arguments);
}
function existsSync() {
return fs.existsSync.apply(fs.existsSync, arguments);
}
function access() {
return fs.access.apply(fs.access, arguments);
}
function removeSync() {
return fs.removeSync.apply(fs.removeSync, arguments);
}
@ -67,18 +83,22 @@ function createWriteStream() {
}
module.exports = {
mkdirpSync: mkdirpSync,
mkdirpSync,
mkdirp,
copySync: copySync,
copy,
copySync,
readFile,
readFileSync: readFileSync,
appendFileSync: appendFileSync,
writeFileSync: writeFileSync,
readJSONSync: readJSONSync,
writeJSONSync: writeJSONSync,
existsSync: existsSync,
removeSync: removeSync,
embarkPath: embarkPath,
dappPath: dappPath,
readFileSync,
appendFileSync,
writeFile,
writeFileSync,
readJSONSync,
writeJson,
writeJSONSync,
access,
existsSync,
removeSync,
embarkPath,
dappPath,
createWriteStream
};

View File

@ -1,57 +1,103 @@
let fs = require('../core/fs.js');
let async = require('async');
var utils = require('../utils/utils.js');
const fs = require('../core/fs.js');
const async = require('async');
const utils = require('../utils/utils.js');
const webpack = require("webpack");
const constants = require('../constants');
const File = require('../core/file');
require("babel-preset-react");
require("babel-preset-es2015");
require("babel-preset-es2016");
require("babel-preset-es2017");
let pipeline;
// Override process.chdir so that we have a partial-implementation PWD for Windows
const realChdir = process.chdir;
process.chdir = (...args) => {
if (!process.env.PWD) {
process.env.PWD = process.cwd();
}
realChdir(...args);
};
class Pipeline {
constructor(options) {
this.buildDir = options.buildDir;
this.contractsFiles = options.contractsFiles;
this.assetFiles = options.assetFiles;
this.events = options.events;
this.logger = options.logger;
this.plugins = options.plugins;
this.pipelinePlugins = options.pipelinePlugins;
this.pluginImports = options.pluginImports;
this.web3Location = options.web3Location;
this.providerCode = options.providerCode;
this.interceptLogs();
}
interceptLogs() {
const context = {};
context.console = console;
context.console.log = this.log.bind(this, 'log');
context.console.warn = this.log.bind(this, 'warn');
context.console.info = this.log.bind(this, 'info');
context.console.debug = this.log.bind(this, 'debug');
context.console.trace = this.log.bind(this, 'trace');
context.console.dir = this.log.bind(this, 'dir');
}
log(type, ...messages) {
process.send({result: constants.pipeline.log, message: messages, type});
}
build(abi, contractsJSON, path, callback) {
let self = this;
const importsList = {};
this.buildContracts(contractsJSON);
self.buildWeb3JS(function() {
let importsList = {};
async.waterfall([
function buildTheContracts(next) {
self.buildContracts(contractsJSON, next);
},
function buildWeb3(next) {
self.buildWeb3JS(next);
},
function createImportList(next) {
importsList["Embark/EmbarkJS"] = fs.dappPath(".embark", 'embark.js');
importsList["Embark/web3"] = fs.dappPath(".embark", 'web3_instance.js');
self.plugins.getPluginsProperty('imports', 'imports').forEach(function (importObject) {
self.pluginImports.forEach(function (importObject) {
let [importName, importLocation] = importObject;
importsList[importName] = importLocation;
});
for (let contractName in contractsJSON) {
let contractCode = self.buildContractJS(contractName);
next();
},
function writeContracts(next) {
async.each(Object.keys(contractsJSON), (contractName, eachCb) => {
self.buildContractJS(contractName, (err, contractCode) => {
let filePath = fs.dappPath(".embark", contractName + '.js');
fs.writeFileSync(filePath, contractCode);
importsList["Embark/contracts/" + contractName] = filePath;
}
fs.writeFile(filePath, contractCode, eachCb);
});
}, next);
},
function assetFileWrite(next) {
// limit:1 due to issues when downloading required files such as web3.js
async.eachOfLimit(self.assetFiles, 1, function (files, targetFile, cb) {
// limit:1 due to issues when downloading required files such as web3.js
async.mapLimit(files, 1,
function(file, fileCb) {
self.logger.trace("reading " + file.filename);
function (file, fileCb) {
file = new File(file); // Re-instantiate a File as through the process, we lose its prototype
console.trace("reading " + file.filename);
if (file.filename.indexOf('.js') >= 0) {
if (file.filename.indexOf('.js') < 0) {
return file.content(function (fileContent) {
self.runPlugins(file, fileContent, fileCb);
});
}
// JS files
let realCwd;
async.waterfall([
@ -66,26 +112,6 @@ class Pipeline {
next();
},
//function findImportsPhase2(next) {
// console.log("====> findImports_2");
// self.webpackRun(file.filename, {
// externals: function(context, request, callback) {
// if (request === utils.joinPath(fs.dappPath(), file.filename)) {
// callback();
// } else {
// //if (request.indexOf('Embark/contracts/') === 0) {
// // let contractName = request.split('/')[2];
// // let contractCode = self.buildContractJS(contractName);
// // let filePath = utils.joinPath(fs.dappPath(), ".embark", contractName + '.js');
// // fs.writeFileSync(filePath, contractCode);
// // importsList[request] = filePath;
// //}
// callback(null, "amd " + Math.random());
// }
// }
// }, true, importsList, next);
//},
function runWebpack(next) {
self.webpackRun(file.filename, {}, true, importsList, true, next);
},
@ -93,33 +119,47 @@ class Pipeline {
function changeCwdBack(next) {
process.chdir(realCwd);
next();
},
function checkFile(next) {
fs.access('./.embark/' + file.filename, (err) => {
if (err) {
console.error("couldn't find file: " + file.filename);
return next("couldn't find file: " + file.filename);
}
next();
});
},
function readFile(next) {
fs.readFile('./.embark/' + file.filename, (err, data) => {
if (err) {
return next(err);
}
next(null, data.toString());
});
},
function runPluginsOnContent(fileContent, next) {
self.runPlugins(file, fileContent, next);
}
], function(err, _result) {
], function (err, contentFile) {
if (err) {
process.chdir(realCwd);
self.logger.error(err);
console.error(err);
return fileCb(err);
}
if (!fs.existsSync('./.embark/' + file.filename)) {
self.logger.error("couldn't find file: " + file.filename);
return fileCb("couldn't find file: " + file.filename);
}
let fileContent = fs.readFileSync('./.embark/' + file.filename).toString();
self.runPlugins(file, fileContent, fileCb);
fileCb(null, contentFile);
});
} else {
file.content(function(fileContent) {
self.runPlugins(file, fileContent, fileCb);
});
}
},
function (err, contentFiles) {
if (err) {
self.logger.warn('errors found while generating ' + targetFile);
console.error('errors found while generating ' + targetFile);
}
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace("creating dir " + self.buildDir + dir);
console.trace("creating dir " + self.buildDir + dir);
fs.mkdirpSync(self.buildDir + dir);
// if it's a directory
@ -130,13 +170,15 @@ class Pipeline {
targetDir = targetDir + '/';
}
contentFiles.map(function (file) {
async.each(contentFiles, function (file, mapCb) {
let filename = file.filename.replace(file.basedir + '/', '');
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
console.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copy(file.path, self.buildDir + targetDir + filename, {overwrite: true}, mapCb);
}, cb);
return;
}
fs.copySync(file.path, self.buildDir + targetDir + filename, {overwrite: true});
});
} else {
let content = contentFiles.map(function (file) {
if (file === undefined) {
return "";
@ -144,27 +186,29 @@ class Pipeline {
return file.content;
}).join("\n");
self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
fs.writeFileSync(self.buildDir + targetFile, content);
}
cb();
console.info("writing file " + (self.buildDir + targetFile).bold.dim);
fs.writeFile(self.buildDir + targetFile, content, cb);
}
);
},
function (_err, _results) {
callback();
});
});
next);
}
], callback);
}
runPlugins(file, fileContent, fileCb) {
const self = this;
let pipelinePlugins = self.plugins.getPluginsFor('pipeline');
if (pipelinePlugins.length <= 0) {
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
if (self.pipelinePlugins.length <= 0) {
return fileCb(null, {
content: fileContent,
filename: file.filename,
path: file.path,
basedir: file.basedir,
modified: true
});
}
async.eachSeries(pipelinePlugins,
function(plugin, pluginCB) {
async.eachSeries(self.pipelinePlugins,
function (plugin, pluginCB) {
if (file.options && file.options.skipPipeline) {
return pluginCB();
}
@ -175,9 +219,15 @@ class Pipeline {
},
function (err) {
if (err) {
self.logger.error(err.message);
console.error(err.message);
}
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, basedir: file.basedir, modified: true});
return fileCb(null, {
content: fileContent,
filename: file.filename,
path: file.path,
basedir: file.basedir,
modified: true
});
}
);
}
@ -197,7 +247,7 @@ class Pipeline {
fs.dappPath('node_modules')
]
},
externals: function(context, request, callback) {
externals: function (context, request, callback) {
callback();
}
};
@ -233,29 +283,39 @@ class Pipeline {
};
}
webpack(webpackOptions).run((_err, _stats) => {
webpack(webpackOptions).run((err, stats) => {
if (err) {
console.error(err);
}
if (!detectErrors) {
return callback();
}
if (_stats.hasErrors()) {
return callback(_stats.toJson().errors.join("\n"));
if (stats.hasErrors()) {
return callback(stats.toJson().errors.join("\n"));
}
callback();
});
}
buildContracts(contractsJSON) {
fs.mkdirpSync(fs.dappPath(this.buildDir, 'contracts'));
for (let className in contractsJSON) {
buildContracts(contractsJSON, callback) {
fs.mkdirp(fs.dappPath(this.buildDir, 'contracts'), (err) => {
if (err) {
return callback(err);
}
async.each(Object.keys(contractsJSON), (className, eachCb) => {
let contract = contractsJSON[className];
fs.writeJSONSync(fs.dappPath(this.buildDir, 'contracts', className + ".json"), contract, {spaces: 2});
}
fs.writeJson(fs.dappPath(this.buildDir, 'contracts', className + ".json"), contract, {spaces: 2}, eachCb);
}, callback);
});
}
buildContractJS(contractName) {
let contractJSON = fs.readFileSync(fs.dappPath(this.buildDir, 'contracts', contractName + '.json')).toString();
buildContractJS(contractName, callback) {
fs.readFile(fs.dappPath(this.buildDir, 'contracts', contractName + '.json'), (err, contractJSON) => {
if (err) {
return callback(err);
}
contractJSON = contractJSON.toString();
let contractCode = "";
contractCode += "import web3 from 'Embark/web3';\n";
@ -268,8 +328,8 @@ class Pipeline {
contractCode += "\n});\n";
contractCode += "export default " + contractName + ";\n";
return contractCode;
callback(null, contractCode);
});
}
buildWeb3JS(cb) {
@ -277,44 +337,47 @@ class Pipeline {
let code = "";
async.waterfall([
function getWeb3Location(next) {
self.events.request("version:get:web3", function(web3Version) {
if (web3Version === "1.0.0-beta") {
return next(null, utils.joinPath(fs.embarkPath("js/web3-1.0.min.js")));
} else {
self.events.request("version:getPackageLocation", "web3", web3Version, function(err, location) {
return next(null, fs.dappPath(location));
});
}
});
},
function getImports(web3Location, next) {
web3Location = web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
code += "\nimport Web3 from '" + web3Location + "';\n";
function getImports(next) {
self.web3Location = self.web3Location.replace(/\\/g, '/'); // Import paths must always have forward slashes
code += "\nimport Web3 from '" + self.web3Location + "';\n";
code += "\n if (typeof web3 !== 'undefined') {";
code += "\n } else {";
code += "\n var web3 = new Web3();\n";
code += "\n }";
self.events.request('provider-code', function(providerCode) {
code += providerCode;
code += self.providerCode;
code += "\nglobal.__embarkContext = __mainContext.__loadManagerInstance;\n";
code += "\nwindow.web3 = web3;\n";
code += "\nexport default web3;\n";
next();
},
function makeDirectory(next) {
fs.mkdirp(fs.dappPath(".embark"), (err, _result) => {
next(err);
});
},
function writeFile(next) {
fs.mkdirpSync(fs.dappPath(".embark"));
fs.writeFileSync(fs.dappPath(".embark", 'web3_instance.js'), code);
next();
fs.writeFile(fs.dappPath(".embark", 'web3_instance.js'), code, next);
}
], function(_err, _result) {
cb();
});
], cb);
}
}
module.exports = Pipeline;
process.on('message', (msg) => {
if (msg.action === constants.pipeline.init) {
pipeline = new Pipeline(msg.options);
return process.send({result: constants.pipeline.initiated});
}
if (msg.action === constants.pipeline.build) {
return pipeline.build(msg.abi, msg.contractsJSON, msg.path, (err) => {
process.send({result: constants.pipeline.built, error: err});
});
}
});
process.on('exit', () => {
process.exit(0);
});