move old pipeline to its own module; generate basic artifacts

This commit is contained in:
Iuri Matias 2019-07-11 17:58:06 -04:00
parent bf91543b20
commit 02f58e069b
12 changed files with 327 additions and 303 deletions

View File

@ -62,7 +62,7 @@ export class ProcessLauncher {
return self.exitCallback(code);
}
if (code) {
this.logger.info(`Child Process ${this.name} exited with code ${code}`);
self.logger.info(`Child Process ${this.name} exited with code ${code}`);
}
});
}

View File

@ -211,7 +211,7 @@ class ContractDeployer {
}
contractAlreadyDeployed(contract, trackedContract, callback) {
console.dir("contractAlreadyDeployed")
console.dir("--> contractAlreadyDeployed")
this.logFunction(contract)(contract.className.bold.cyan + __(" already deployed at ").green + trackedContract.address.bold.cyan);
contract.deployedAddress = trackedContract.address;
this.events.emit("deploy:contract:deployed", contract);
@ -363,7 +363,7 @@ class ContractDeployer {
return next(err);
}
next(null, receipt);
// });
});
}, hash => {
self.logFunction(contract)(__("deploying") + " " + contract.className.bold.cyan + " " + __("with").green + " " + contract.gas + " " + __("gas at the price of").green + " " + contract.gasPrice + " " + __("Wei, estimated cost:").green + " " + estimatedCost + " Wei".green + " (txHash: " + hash.bold.cyan + ")");
});

View File

@ -53,8 +53,7 @@
"embark-i18n": "^4.1.0-beta.1",
"embark-utils": "^4.1.0-beta.2",
"find-up": "2.1.0",
"fs-extra": "7.0.1",
"webpack": "4.29.3"
"fs-extra": "7.0.1"
},
"devDependencies": {
"@babel/cli": "7.2.3",

View File

@ -1,9 +1,5 @@
const async = require('async');
import { __ } from 'embark-i18n';
import { dappPath, joinPath, LongRunningProcessTimer } from 'embark-utils';
import { ProcessLauncher } from 'embark-core';
const constants = require('embark-core/constants');
const WebpackConfigReader = require('./webpackConfigReader');
import { dappPath } from 'embark-utils';
const PipelineAPI = require('./api.js');
@ -13,37 +9,18 @@ const PipelineAPI = require('./api.js');
class Pipeline {
constructor(embark, options) {
this.embark = embark;
this.env = embark.config.env;
this.buildDir = embark.config.buildDir;
this.contractsFiles = embark.config.contractsFiles;
this.assetFiles = embark.config.assetFiles;
this.embarkConfig = embark.config.embarkConfig;
this.events = embark.events;
this.logger = embark.config.logger;
this.plugins = embark.config.plugins;
this.fs = embark.fs;
this.webpackConfigName = options.webpackConfigName;
this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
this.pipelineConfig = embark.config.pipelineConfig;
this.isFirstBuild = true;
this.useDashboard = options.useDashboard;
// track changes to the pipeline config in the filesystem
this.events.on('config:load:pipeline', (pipelineConfig) => {
this.pipelineConfig = pipelineConfig;
});
this.events.setCommandHandler('pipeline:generateAll', (cb) => {
this.generateAll(cb);
});
this.events.setCommandHandler('pipeline:build', (options, callback) => {
if (!this.pipelineConfig.enabled) {
return this.buildContracts([], callback);
}
this.build(options, callback);
});
this.events.setCommandHandler('pipeline:build:contracts', callback => this.buildContracts([], callback));
// this.events.setCommandHandler('pipeline:build', (options, callback) => {
// if (!this.pipelineConfig.enabled) {
// return this.buildContracts([], callback);
// }
// this.build(options, callback);
// });
// this.events.setCommandHandler('pipeline:build:contracts', callback => this.buildContracts([], callback));
// TODO: action in the constructor, shoudn't be happening..
// this.fs.removeSync(this.buildDir);
@ -52,6 +29,10 @@ class Pipeline {
this.files = {}
this.events.setCommandHandler('pipeline:generateAll', (cb) => {
this.generateAll(cb);
});
this.events.setCommandHandler('pipeline:register', (params, cb) => {
this.files[dappPath(...params.path, params.file)] = params;
if (cb) {
@ -124,268 +105,6 @@ class Pipeline {
});
}
// =================
// =================
// =================
// =================
// =================
build({modifiedAssets}, callback) {
let self = this;
const importsList = {};
let placeholderPage;
const contractsDir = dappPath(self.embarkConfig.generationDir, constants.dappArtifacts.contractsJs);
if (!self.assetFiles || !Object.keys(self.assetFiles).length) {
return self.buildContracts([], callback);
}
async.waterfall([
// TODO: doesn't seem to be actually used (it's done on the webserver itself)
function createPlaceholderPage(next) {
if (self.isFirstBuild) {
self.isFirstBuild = false;
return next();
}
self.events.request('placeholder:build', next);
},
(next) => self.buildContracts(importsList, next),
function createImportList(next) {
importsList["Embark/EmbarkJS"] = dappPath(self.embarkConfig.generationDir, constants.dappArtifacts.embarkjs);
importsList["Embark/contracts"] = contractsDir;
self.plugins.getPluginsProperty('imports', 'imports').forEach(importObject => {
let [importName, importLocation] = importObject;
importsList[importName] = importLocation;
});
next();
},
function shouldRunWebpack(next) {
// assuming we got here because an asset was changed, let's check our webpack config
// to see if the changed asset requires webpack to run
if (!(modifiedAssets && modifiedAssets.length)) return next(null, false);
const configReader = new WebpackConfigReader({webpackConfigName: self.webpackConfigName});
return configReader.readConfig((err, config) => {
if (err) return next(err);
if (typeof config !== 'object' || config === null) {
return next(__('Pipeline: bad webpack config, the resolved config was null or not an object'));
}
const shouldRun = modifiedAssets.some(modifiedAsset => config.module.rules.some(rule => rule.test.test(modifiedAsset)));
return next(null, !shouldRun);
});
},
function runWebpack(shouldNotRun, next) {
if (shouldNotRun) return next();
const assets = Object.keys(self.assetFiles).filter(key => key.match(/\.js$/));
if (!assets || !assets.length) {
return next();
}
let strAssets = '';
if (!self.useDashboard) {
assets.forEach(key => {
strAssets += ('\n ' + (joinPath(self.buildDir, key)).bold.dim);
});
}
const timer = new LongRunningProcessTimer(
self.logger,
'webpack',
'0',
`${'Pipeline:'.cyan} Bundling dapp using '${self.webpackConfigName}' config...${strAssets}`,
`${'Pipeline:'.cyan} Still bundling dapp using '${self.webpackConfigName}' config... ({{duration}})${strAssets}`,
`${'Pipeline:'.cyan} Finished bundling dapp in {{duration}}${strAssets}`,
{
showSpinner: !self.useDashboard,
interval: self.useDashboard ? 5000 : 1000,
longRunningThreshold: 15000
}
);
timer.start();
let built = false;
const webpackProcess = new ProcessLauncher({
embark: self.embark,
plugins: self.plugins,
modulePath: joinPath(__dirname, 'webpackProcess.js'),
logger: self.logger,
events: self.events,
exitCallback: code => {
if (!built) {
return next(`Webpack build exited with code ${code} before the process finished`);
}
if (code) {
self.logger.error(__('Webpack build process exited with code ', code));
}
}
});
webpackProcess.send({
action: constants.pipeline.init,
options: {
webpackConfigName: self.webpackConfigName,
pipelineConfig: self.pipelineConfig,
fs: self.embark.fs
}
});
webpackProcess.send({action: constants.pipeline.build, assets: self.assetFiles, importsList});
webpackProcess.once('result', constants.pipeline.built, (msg) => {
built = true;
webpackProcess.kill();
return next(msg.error);
});
webpackProcess.once('result', constants.pipeline.webpackDone, () => {
timer.end();
});
},
function assetFileWrite(next) {
async.eachOf(
// assetFileWrite should not process .js files
Object.keys(self.assetFiles)
.filter(key => !key.match(/\.js$/))
.reduce((obj, key) => {
obj[key] = self.assetFiles[key];
return obj;
}, {}),
function (files, targetFile, cb) {
const isDir = targetFile.slice(-1) === '/' || targetFile.slice(-1) === '\\' || targetFile.indexOf('.') === -1;
// if it's not a directory
if (!isDir) {
self.logger.info('Pipeline: '.cyan + __("writing file") + " " + (joinPath(self.buildDir, targetFile)).bold.dim);
}
async.map(
files,
function (file, fileCb) {
self.logger.trace("reading " + file.path);
file.content.then((fileContent) => {
self.runPlugins(file, fileContent, fileCb);
}).catch(fileCb);
},
function (err, contentFiles) {
if (err) {
self.logger.error('Pipeline: '.cyan + __('errors found while generating') + ' ' + targetFile);
}
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace(`${'Pipeline:'.cyan} creating dir ` + joinPath(self.buildDir, dir));
self.fs.mkdirpSync(joinPath(self.buildDir, dir));
// if it's a directory
if (isDir) {
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
async.each(contentFiles, function (file, eachCb) {
let filename = file.path.replace(file.basedir + '/', '');
self.logger.info(`${'Pipeline:'.cyan} writing file ` + (joinPath(self.buildDir, targetDir, filename)).bold.dim);
self.fs.copy(file.path, joinPath(self.buildDir, targetDir, filename), {overwrite: true}, eachCb);
}, cb);
return;
}
let content = contentFiles.map(file => {
if (file === undefined) {
return "";
}
return file.content;
}).join("\n");
if (new RegExp(/^index.html?/i).test(targetFile)) {
targetFile = targetFile.replace('index', 'index-temp');
placeholderPage = targetFile;
}
self.fs.writeFile(joinPath(self.buildDir, targetFile), content, cb);
}
);
},
next
);
},
function removePlaceholderPage(next) {
let placeholderFile = joinPath(self.buildDir, placeholderPage);
self.fs.access(joinPath(self.buildDir, placeholderPage), (err) => {
if (err) return next(); // index-temp doesn't exist, do nothing
// rename index-temp.htm/l to index.htm/l, effectively replacing our placeholder page
// with the contents of the built index.html page
self.fs.move(placeholderFile, placeholderFile.replace('index-temp', 'index'), {overwrite: true}, next);
});
}
], callback);
}
buildContracts(importsList, cb) {
const self = this;
async.waterfall([
function makeDirectory(next) {
self.fs.mkdirp(dappPath(self.buildDir, 'contracts'), err => next(err));
},
function getContracts(next) {
self.events.request('contracts:list', next);
},
function writeContractsJSON(contracts, next) {
async.each(contracts, (contract, eachCb) => {
self.fs.writeJson(dappPath(
self.buildDir,
'contracts', contract.className + '.json'
), contract, {spaces: 2}, eachCb);
}, () => next(null, contracts));
},
function writeContractJS(contracts, next) {
const contractsDir = dappPath(self.embarkConfig.generationDir, constants.dappArtifacts.contractsJs);
self.fs.mkdirp(contractsDir, err => {
if (err) return next(err);
// Create a file index.js that requires all contract files
// Used to enable alternate import syntax:
// e.g. import {Token} from 'Embark/contracts'
// e.g. import * as Contracts from 'Embark/contracts'
let importsHelperFile = self.fs.createWriteStream(joinPath(contractsDir, 'index.js'));
importsHelperFile.write('module.exports = {\n');
async.eachOf(contracts, (contract, idx, eachCb) => {
self.events.request('code-generator:contract', contract.className, (err, contractPath) => {
if (err) {
return eachCb(err);
}
importsList["Embark/contracts/" + contract.className] = dappPath(contractPath);
// add the contract to the exports list to support alternate import syntax
importsHelperFile.write(`"${contract.className}": require('./${contract.className}').default,\n`);
eachCb();
});
}, () => {
importsHelperFile.write('\n};'); // close the module.exports = {}
importsHelperFile.close(next); // close the write stream
});
});
}
], cb);
}
runPlugins(file, fileContent, fileCb) {
const self = this;
if (self.pipelinePlugins.length <= 0) {
return fileCb(null, {content: fileContent, path: file.path, basedir: file.basedir, modified: true});
}
async.eachSeries(self.pipelinePlugins, (plugin, pluginCB) => {
if (file.options && file.options.skipPipeline) {
return pluginCB();
}
fileContent = plugin.runPipeline({targetFile: file.path, source: fileContent});
file.modified = true;
pluginCB();
}, err => {
if (err) {
self.logger.error(err.message);
}
return fileCb(null, {content: fileContent, path: file.path, basedir: file.basedir, modified: true});
});
}
}
module.exports = Pipeline;

View File

@ -10,8 +10,8 @@ module.exports = (api) => {
const node = cloneDeep(base);
Object.assign(node, {
ignore: [
'src/lib/modules/pipeline/babel-loader-overrides.js',
'src/lib/modules/pipeline/webpack.config.js'
'src/lib/modules/basic-pipeline/babel-loader-overrides.js',
'src/lib/modules/basic-pipeline/webpack.config.js'
]
});

View File

@ -142,7 +142,9 @@ class Engine {
pipelineService(_options) {
const self = this;
this.registerModulePackage('embark-pipeline', {
this.registerModulePackage('embark-pipeline', { plugins: this.plugins });
this.registerModule('basic-pipeline', {
plugins: this.plugins,
webpackConfigName: this.webpackConfigName,
useDashboard: this.useDashboard
});

View File

@ -0,0 +1,230 @@
const async = require('async');
import { __ } from 'embark-i18n';
import { ProcessLauncher } from 'embark-core';
import { dappPath, joinPath, LongRunningProcessTimer } from 'embark-utils';
const constants = require('embark-core/constants');
const WebpackConfigReader = require('./webpackConfigReader');
class BasicPipeline {
constructor(embark, options) {
this.embark = embark;
this.assetFiles = embark.config.assetFiles;
this.isFirstBuild = true;
this.embarkConfig = embark.config.embarkConfig;
// TODO: why god why
this.useDashboard = options.useDashboard;
this.fs = embark.fs;
this.webpackConfigName = options.webpackConfigName;
this.env = embark.config.env;
this.buildDir = embark.config.buildDir;
this.contractsFiles = embark.config.contractsFiles;
this.embarkConfig = embark.config.embarkConfig;
this.logger = embark.logger;
this.events = embark.events;
this.plugins = options.plugins;
this.pipelinePlugins = this.plugins.getPluginsFor('pipeline');
this.pipelineConfig = embark.config.pipelineConfig;
let plugin = this.plugins.createPlugin('basic-pipeline', {});
plugin.registerActionForEvent("pipeline:generateAll:after", this.webpackAssets.bind(this));
// track changes to the pipeline config in the filesystem
this.events.on('config:load:pipeline', (pipelineConfig) => {
this.pipelineConfig = pipelineConfig;
});
}
webpackAssets(params, done) {
console.dir("=======================> webpackAssets");
let self = this;
let placeholderPage;
const importsList = {};
if (!self.assetFiles || !Object.keys(self.assetFiles).length) {
return done(); // no assetFiles so nothing to do
}
let modifiedAssets = self.assetFiles;
async.waterfall([
(next) => {
console.dir("=======================> importsList");
importsList["Embark/EmbarkJS"] = dappPath(self.embarkConfig.generationDir, 'embarkjs.js');
importsList["Embark/contracts"] = this.embarkConfig.generationDir;
next();
},
function shouldRunWebpack(next) {
console.dir("=======================> shouldRunWebpack");
// assuming we got here because an asset was changed, let's check our webpack config
// to see if the changed asset requires webpack to run
if (!(modifiedAssets && modifiedAssets.length)) return next(null, false);
const configReader = new WebpackConfigReader({webpackConfigName: self.webpackConfigName});
return configReader.readConfig((err, config) => {
if (err) return next(err);
if (typeof config !== 'object' || config === null) {
return next(__('Pipeline: bad webpack config, the resolved config was null or not an object'));
}
const shouldRun = modifiedAssets.some(modifiedAsset => config.module.rules.some(rule => rule.test.test(modifiedAsset)));
return next(null, !shouldRun);
});
},
function runWebpack(shouldNotRun, next) {
console.dir("=======================> runWebpack");
if (shouldNotRun) return next();
const assets = Object.keys(self.assetFiles).filter(key => key.match(/\.js$/));
if (!assets || !assets.length) {
return next();
}
let strAssets = '';
if (!self.useDashboard) {
assets.forEach(key => {
strAssets += ('\n ' + (joinPath(self.buildDir, key)).bold.dim);
});
}
const timer = new LongRunningProcessTimer(
self.logger,
'webpack',
'0',
`${'Pipeline:'.cyan} Bundling dapp using '${self.webpackConfigName}' config...${strAssets}`,
`${'Pipeline:'.cyan} Still bundling dapp using '${self.webpackConfigName}' config... ({{duration}})${strAssets}`,
`${'Pipeline:'.cyan} Finished bundling dapp in {{duration}}${strAssets}`,
{
showSpinner: !self.useDashboard,
interval: self.useDashboard ? 5000 : 1000,
longRunningThreshold: 15000
}
);
timer.start();
let built = false;
const webpackProcess = new ProcessLauncher({
embark: self.embark,
plugins: self.plugins,
modulePath: joinPath(__dirname, 'webpackProcess.js'),
logger: self.logger,
events: self.events,
exitCallback: code => {
if (!built) {
return next(`Webpack build exited with code ${code} before the process finished`);
}
if (code) {
self.logger.error(__('Webpack build process exited with code ', code));
}
}
});
webpackProcess.send({
action: constants.pipeline.init,
options: {
webpackConfigName: self.webpackConfigName,
pipelineConfig: self.pipelineConfig,
fs: self.embark.fs
}
});
webpackProcess.send({action: constants.pipeline.build, assets: self.assetFiles, importsList});
webpackProcess.once('result', constants.pipeline.built, (msg) => {
built = true;
webpackProcess.kill();
return next(msg.error);
});
webpackProcess.once('result', constants.pipeline.webpackDone, () => {
timer.end();
});
},
function assetFileWrite(next) {
console.dir("=======================> assetFileWrite 136");
async.eachOf(
// assetFileWrite should not process .js files
Object.keys(self.assetFiles)
.filter(key => !key.match(/\.js$/))
.reduce((obj, key) => {
obj[key] = self.assetFiles[key];
return obj;
}, {}),
function (files, targetFile, cb) {
console.dir("== eachOf: " + targetFile);
const isDir = targetFile.slice(-1) === '/' || targetFile.slice(-1) === '\\' || targetFile.indexOf('.') === -1;
// if it's not a directory
if (!isDir) {
self.logger.info('Pipeline: '.cyan + __("_1_ writing file") + " " + (joinPath(self.buildDir, targetFile)).bold.dim);
}
console.dir("async.map")
async.map(
files,
function (file, fileCb) {
self.logger.trace("reading " + file.path);
console.dir("reading " + file.path);
file.content.then((fileContent) => {
self.runPlugins(file, fileContent, fileCb);
}).catch(fileCb);
},
function (err, contentFiles) {
try {
if (err) {
self.logger.error('Pipeline: '.cyan + __('errors found while generating') + ' ' + targetFile);
}
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace(`${'Pipeline:'.cyan} creating dir ` + joinPath(self.buildDir, dir));
self.fs.mkdirpSync(joinPath(self.buildDir, dir));
// if it's a directory
if (isDir) {
console.dir("---> isDir")
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
async.each(contentFiles, function (file, eachCb) {
let filename = file.path.replace(file.basedir + '/', '');
self.logger.info(`${'Pipeline:'.cyan} __ writing file ` + (joinPath(self.buildDir, targetDir, filename)).bold.dim);
self.fs.copy(file.path, joinPath(self.buildDir, targetDir, filename), {overwrite: true}, eachCb);
}, cb);
return;
}
console.dir("---> not a dir")
let content = contentFiles.map(file => {
if (file === undefined) {
return "";
}
return file.content;
}).join("\n");
if (new RegExp(/^index.html?/i).test(targetFile)) {
targetFile = targetFile.replace('index', 'index-temp');
placeholderPage = targetFile;
}
console.dir("--> writing file")
self.fs.writeFile(joinPath(self.buildDir, targetFile), content, cb);
} catch(err) {
console.dir(err)
}
}
);
},
next
);
},
function removePlaceholderPage(next) {
console.dir("=======================> removePlaceholderPage");
let placeholderFile = joinPath(self.buildDir, placeholderPage);
self.fs.access(joinPath(self.buildDir, placeholderPage), (err) => {
if (err) return next(); // index-temp doesn't exist, do nothing
// rename index-temp.htm/l to index.htm/l, effectively replacing our placeholder page
// with the contents of the built index.html page
self.fs.move(placeholderFile, placeholderFile.replace('index-temp', 'index'), {overwrite: true}, next);
});
}
], done);
}
}
module.exports = BasicPipeline;

View File

@ -1,4 +1,6 @@
require('ejs');
import { embarkPath } from "embark-utils";
import { transform } from "@babel/core";
const Templates = {
vanilla_contract: require('./vanilla-contract.js.ejs')
@ -17,6 +19,7 @@ class Web3Plugin {
plugin.registerActionForEvent("deploy:contract:deployed", this.registerInVm.bind(this));
plugin.registerActionForEvent("deploy:contract:deployed", this.addContractJSONToPipeline.bind(this));
plugin.registerActionForEvent("deploy:contract:deployed", this.addContractFileToPipeline.bind(this));
plugin.registerActionForEvent("pipeline:generateAll:before", this.addEmbarkJSNode.bind(this));
plugin.registerActionForEvent("pipeline:generateAll:before", this.addContractIndexToPipeline.bind(this));
}
@ -54,6 +57,9 @@ class Web3Plugin {
addContractFileToPipeline(params, cb) {
const contract = params.contract;
const contractName = contract.className;
console.dir("--------------");
console.dir("--------------");
console.dir(contract.className);
const contractJSON = contract.abiDefinition;
const contractCode = `
@ -85,8 +91,9 @@ class Web3Plugin {
addContractIndexToPipeline(_params, cb) {
this.events.request("contracts:list", (err, contracts) => {
let imports = contracts.map((c) => {
return `"${c.className}": require('./${c.className}').default)`;
contracts.forEach(console.dir)
let imports = contracts.filter(c => c.deployedAddress || c.deploy).map((c) => {
return `"${c.className}": require('./${c.className}').default`;
}).join(",\n");
let code = 'module.exports = {\n';
@ -102,6 +109,73 @@ class Web3Plugin {
});
}
// TODO: ideally shouldn't be done here
addEmbarkJSNode(_params, cb) {
let embarkjsCode = '';
// TODO: the symblink is unclear at this point, but if needed, it should be done at the pipeline through a request
// TODO: embarkjs stuff should also be in a embark-embarkjs module
// self.generateSymlink(location, 'embarkjs', (err, symlinkDest) => {
// if (err) {
// self.logger.error(__('Error creating a symlink to EmbarkJS'));
// return next(err);
// }
// embarkjsCode += `\nconst EmbarkJS = require("${symlinkDest}").default || require("${symlinkDest}");`;
embarkjsCode += `\nconst EmbarkJS = require("embarkjs").default;`;
// embarkjsCode += `\nEmbarkJS.environment = '${self.env}';`;
embarkjsCode += "\nglobal.EmbarkJS = EmbarkJS;";
// next();
// });
let code = "";
code += "\n" + embarkjsCode + "\n";
code += "\nexport default EmbarkJS;";
code += "\nif (typeof module !== 'undefined' && module.exports) {" +
"\n\tmodule.exports = EmbarkJS;" +
"\n}";
code += '\n/* eslint-enable */\n';
// TODO: should be done in async.waterfall
this.events.request("pipeline:register", {
path: [this.embarkConfig.generationDir],
file: 'embarkjs.js',
format: 'js',
content: code
});
// embark.js
// self.generateArtifact(code, constants.dappArtifacts.embarkjs, '', next);
transform(code, {
cwd: embarkPath(),
"presets": [
[
"@babel/preset-env", {
"targets": {
"node": "8.11.3"
}
}
]
]
}, (err, result) => {
if (err) {
return cb(err);
}
this.events.request("pipeline:register", {
path: [this.embarkConfig.generationDir],
file: 'embarkjs.node.js',
format: 'js',
content: code
}, cb);
// self.generateArtifact(result.code, constants.dappArtifacts.embarkjsnode, '', next);
});
// embark.node.js
}
}
module.exports = Web3Plugin;