add file type so files are loaded when needed; support to configure web3 and solc versions

This commit is contained in:
Iuri Matias 2017-07-05 08:35:51 -04:00
parent 9b98980efd
commit f45ce963b4
14 changed files with 264 additions and 117 deletions

View File

@ -6,6 +6,7 @@ class Compiler {
constructor(options) { constructor(options) {
this.plugins = options.plugins; this.plugins = options.plugins;
this.logger = options.logger; this.logger = options.logger;
this.solcVersion = options.solcVersion;
} }
compile_contracts(contractFiles, cb) { compile_contracts(contractFiles, cb) {
@ -53,16 +54,28 @@ class Compiler {
let solcW; let solcW;
async.waterfall([ async.waterfall([
function prepareInput(callback) { function prepareInput(callback) {
for (let i = 0; i < contractFiles.length; i++) { async.each(contractFiles,
// TODO: this depends on the config function(file, fileCb) {
let filename = contractFiles[i].filename.replace('app/contracts/', ''); let filename = file.filename.replace('app/contracts/', '');
input[filename] = contractFiles[i].content.toString(); file.content(function(fileContent) {
} input[filename] = fileContent;
callback(); fileCb();
});
},
function (err) {
callback(err);
}
);
//for (let i = 0; i < contractFiles.length; i++) {
// // TODO: this depends on the config
// let filename = contractFiles[i].filename.replace('app/contracts/', '');
// input[filename] = contractFiles[i].content.toString();
//}
//callback();
}, },
function loadCompiler(callback) { function loadCompiler(callback) {
// TODO: there ino need to load this twice // TODO: there ino need to load this twice
solcW = new SolcW(); solcW = new SolcW(self.solcVersion);
if (solcW.isCompilerLoaded()) { if (solcW.isCompilerLoaded()) {
return callback(); return callback();
} }

View File

@ -12,6 +12,7 @@ class ContractsManager {
this.contracts = {}; this.contracts = {};
this.logger = options.logger; this.logger = options.logger;
this.plugins = options.plugins; this.plugins = options.plugins;
this.solcVersion = options.contractsConfig.versions["solc"];
this.contractDependencies = {}; this.contractDependencies = {};
} }
@ -20,7 +21,7 @@ class ContractsManager {
let self = this; let self = this;
async.waterfall([ async.waterfall([
function compileContracts(callback) { function compileContracts(callback) {
let compiler = new Compiler({plugins: self.plugins, logger: self.logger}); let compiler = new Compiler({plugins: self.plugins, logger: self.logger, solcVersion: self.solcVersion});
compiler.compile_contracts(self.contractFiles, function (err, compiledObject) { compiler.compile_contracts(self.contractFiles, function (err, compiledObject) {
self.compiledContracts = compiledObject; self.compiledContracts = compiledObject;
callback(err); callback(err);

View File

@ -2,7 +2,9 @@ let solc;
process.on('message', function (msg) { process.on('message', function (msg) {
if (msg.action === 'loadCompiler') { if (msg.action === 'loadCompiler') {
solc = require('solc'); //solc = require('solc');
//console.log("requiring compiler at " + msg.solcLocation);
solc = require(msg.solcLocation);
process.send({result: "loadedCompiler"}); process.send({result: "loadedCompiler"});
} }

View File

@ -1,9 +1,15 @@
let utils = require('../utils/utils.js'); let utils = require('../utils/utils.js');
let solcProcess; let solcProcess;
let compilerLoaded = false; let compilerLoaded = false;
var npm = require('../pipeline/npm.js');
let path = require('path');
class SolcW { class SolcW {
constructor(version) {
this.solcVersion = version;
}
load_compiler(done) { load_compiler(done) {
if (compilerLoaded) { if (compilerLoaded) {
done(); done();
@ -16,7 +22,13 @@ class SolcW {
compilerLoaded = true; compilerLoaded = true;
done(); done();
}); });
solcProcess.send({action: 'loadCompiler'}); npm.getPackageVersion('solc', '0.4.10', false, function(location) {
console.log("new compiler installed at " + location);
//let requirePath = path.join(process.env.PWD, location.substr(2));
let requirePath = path.join(process.env.PWD, location);
console.log(requirePath);
solcProcess.send({action: 'loadCompiler', solcLocation: requirePath});
});
} }
isCompilerLoaded() { isCompilerLoaded() {

View File

@ -1,4 +1,5 @@
var fs = require('./fs.js'); var fs = require('./fs.js');
var File = require('./file.js');
var Plugins = require('./plugins.js'); var Plugins = require('./plugins.js');
var utils = require('../utils/utils.js'); var utils = require('../utils/utils.js');
var npm = require('../pipeline/npm.js'); var npm = require('../pipeline/npm.js');
@ -45,9 +46,9 @@ Config.prototype.loadConfigFiles = function(options) {
this.loadStorageConfigFile(); this.loadStorageConfigFile();
this.loadCommunicationConfigFile(); this.loadCommunicationConfigFile();
this.loadContractsConfigFile();
this.loadPipelineConfigFile(); this.loadPipelineConfigFile();
this.loadContractsConfigFile();
this.loadWebServerConfigFile(); this.loadWebServerConfigFile();
this.loadChainTrackerFile(); this.loadChainTrackerFile();
this.loadPluginContractFiles(); this.loadPluginContractFiles();
@ -58,8 +59,8 @@ Config.prototype.reloadConfig = function() {
this.loadBlockchainConfigFile(); this.loadBlockchainConfigFile();
this.loadStorageConfigFile(); this.loadStorageConfigFile();
this.loadCommunicationConfigFile(); this.loadCommunicationConfigFile();
this.loadPipelineConfigFile();
this.loadContractsConfigFile(); this.loadContractsConfigFile();
this.loadPipelineConfigFile();
this.loadChainTrackerFile(); this.loadChainTrackerFile();
}; };
@ -234,34 +235,38 @@ Config.prototype.loadFiles = function(files) {
if (file === 'embark.js') { if (file === 'embark.js') {
if (self.blockchainConfig.enabled || self.communicationConfig.provider === 'whisper' || self.communicationConfig.available_providers.indexOf('whisper') >= 0) { if (self.blockchainConfig.enabled || self.communicationConfig.provider === 'whisper' || self.communicationConfig.available_providers.indexOf('whisper') >= 0) {
readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")}); let web3Version = self.contractsConfig.versions["web3.js"];
if (web3Version) {
//if (false) {
//readFiles.push(new File({filename: 'web3-' + web3Version + '.js', type: 'custom', resolver: function(callback) {
readFiles.push(new File({filename: 'web3.js', type: 'custom', resolver: function(callback) {
npm.getPackageVersion('web3', web3Version, 'dist/web3.js', function(web3Content) {
callback(web3Content);
});
}}));
} else {
readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"}));
}
} }
if (self.storageConfig.enabled && (self.storageConfig.provider === 'ipfs' || self.storageConfig.available_providers.indexOf('ipfs') >= 0)) { if (self.storageConfig.enabled && (self.storageConfig.provider === 'ipfs' || self.storageConfig.available_providers.indexOf('ipfs') >= 0)) {
readFiles.push({filename: 'ipfs.js', content: fs.readFileSync(fs.embarkPath("js/ipfs.js")).toString(), path: fs.embarkPath("js/ipfs.js")}); readFiles.push(new File({filename: 'ipfs.js', type: 'embark_internal', path: "js/ipfs.js"}));
} }
if (self.communicationConfig.enabled && (self.communicationConfig.provider === 'orbit' || self.communicationConfig.available_providers.indexOf('orbit') >= 0)) { if (self.communicationConfig.enabled && (self.communicationConfig.provider === 'orbit' || self.communicationConfig.available_providers.indexOf('orbit') >= 0)) {
// TODO: remove duplicated files if functionality is the same for storage and orbit // TODO: remove duplicated files if functionality is the same for storage and orbit
readFiles.push({filename: 'ipfs-api.js', content: fs.readFileSync(fs.embarkPath("js/ipfs-api.min.js")).toString(), path: fs.embarkPath("js/ipfs-api.min.js")}); readFiles.push(new File({filename: 'ipfs-api.js', type: 'embark_internal', path: "js/ipfs-api.min.js"}));
readFiles.push({filename: 'orbit.js', content: fs.readFileSync(fs.embarkPath("js/orbit.min.js")).toString(), path: fs.embarkPath("js/orbit.min.js")}); readFiles.push(new File({filename: 'orbit.js', type: 'embark_internal', path: "js/orbit.min.js"}));
} }
readFiles.push({filename: 'embark.js', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")}); readFiles.push(new File({filename: 'embark.js', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
} }
if (file === '$EMBARK_JS') { if (file === '$EMBARK_JS') {
readFiles.push({filename: '$EMBARK_JS', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")}); readFiles.push(new File({filename: '$EMBARK_JS', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
}
if (file.indexOf("web3-") === 0) {
let web3Version = file.split('web3-')[1].split(".js")[0];
npm.getPackageVersion('web3', web3Version, function(web3Content) {
self.logger.error('downloaded web3');
readFiles.push({filename: file, content: web3Content, path: fs.embarkPath("js/web3.js")});
});
}
if (file === "web3.js") {
readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")});
} }
//if (file === "web3.js") {
// readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"));
//}
}); });
// get plugins // get plugins
@ -298,13 +303,13 @@ Config.prototype.loadFiles = function(files) {
return; return;
} else if (file.indexOf("web3") === 0) { } else if (file.indexOf("web3") === 0) {
return; return;
} else if (file === 'abi.js') { //} else if (file === 'abi.js') {
readFiles.push({filename: file, content: "", path: file}); // readFiles.push({filename: file, content: "", path: file});
} else { } else {
if (file.indexOf('.') >= 0) { if (file.indexOf('.') >= 0) {
readFiles.push({filename: file, content: fs.readFileSync(file).toString(), path: file}); readFiles.push(new File({filename: file, type: "dapp_file", path: file}));
} else if (file[0] === '$') { } else if (file[0] === '$') {
readFiles.push({filename: file, content: "", path: file}); //readFiles.push(new File({filename: file, type: 'embark_internal', path: file}));
} }
} }
}); });
@ -320,7 +325,10 @@ Config.prototype.loadPluginContractFiles = function() {
contractsPlugins.forEach(function(plugin) { contractsPlugins.forEach(function(plugin) {
plugin.contractsFiles.forEach(function(file) { plugin.contractsFiles.forEach(function(file) {
var filename = file.replace('./',''); var filename = file.replace('./','');
self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)}); //self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)});
self.contractsFiles.push(new File({filename: filename, type: 'custom', resolver: function(callback) {
callback(plugin.loadPluginFile(file));
}}));
}); });
}); });
} }

View File

@ -83,8 +83,9 @@ class Engine {
this.events.on('abi', function (abi, contractsJSON) { this.events.on('abi', function (abi, contractsJSON) {
self.currentAbi = abi; self.currentAbi = abi;
self.contractsJSON = contractsJSON; self.contractsJSON = contractsJSON;
pipeline.build(abi, contractsJSON); pipeline.build(abi, contractsJSON, null, function() {
self.events.emit('outputDone'); self.events.emit('outputDone');
});
}); });
// TODO: still need to redeploy contracts because the original contracts // TODO: still need to redeploy contracts because the original contracts
// config is being corrupted // config is being corrupted

26
lib/core/file.js Normal file
View File

@ -0,0 +1,26 @@
let fs = require('./fs.js');
class File {
constructor(options) {
this.filename = options.filename;
this.type = options.type;
this.path = options.path;
this.resolver = options.resolver;
}
content(callback) {
if (this.type === 'embark_internal') {
return callback(fs.readFileSync(fs.embarkPath(this.path)).toString());
} else if (this.type === 'dapp_file') {
return callback(fs.readFileSync(this.path).toString());
} else if (this.type === 'custom') {
return this.resolver(callback);
} else {
throw new Error("unknown file: " + this.filename);
}
}
}
module.exports = File;

View File

@ -34,6 +34,10 @@ function embarkPath(fileOrDir) {
return utils.joinPath(__dirname, '/../../', fileOrDir); return utils.joinPath(__dirname, '/../../', fileOrDir);
} }
function dappPath() {
return process.env.PWD;
}
module.exports = { module.exports = {
mkdirpSync: mkdirpSync, mkdirpSync: mkdirpSync,
copySync: copySync, copySync: copySync,

View File

@ -1,12 +1,16 @@
let utils = require('../utils/utils.js'); let utils = require('../utils/utils.js');
let fs = require('../core/fs.js');
let o_fs = require('fs-extra');
let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
var tar = require('tar');
module.exports = { module.exports = {
// TODO: need to refactor to make it truly generalistic, perhaps by getPackageVersion: function(packageName, version, returnContent, callback) {
// downloading the tarball itself let npmRegistry = "https://registry.npmjs.org/" + packageName + "/" + version;
getPackageVersion: function(packageName, version, callback) {
let npmRegistry = "http://registry.npmjs.org/" + packageName + "/" + version;
utils.httpGet(npmRegistry, function (res) { utils.httpsGet(npmRegistry, function (res) {
let body = ''; let body = '';
res.on('data', function (d) { res.on('data', function (d) {
@ -14,19 +18,53 @@ module.exports = {
}); });
res.on('end', function () { res.on('end', function () {
let registryJSON = JSON.parse(body); let registryJSON = JSON.parse(body);
let gitHash = registryJSON.gitHead;
let repo = registryJSON.homepage.split("github.com/")[1];
let gitUrl = "http://raw.githubusercontent.com/" + repo + "/" + gitHash + "/dist/" + packageName + ".js"; let tarball = registryJSON.dist.tarball;
utils.httpGet(gitUrl, function (res2) {
let body = ''; var download = function(url, dest, cb) {
res2.on('data', function (d) { var file = o_fs.createWriteStream(dest);
body += d; var request = (url.substring(0,5) === 'https' ? https : http).get(url, function(response) {
response.pipe(file);
file.on('finish', function() {
file.close(cb); // close() is async, call cb after close completes.
});
}).on('error', function(err) { // Handle errors
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
}); });
res2.on('end', function () { };
callback(body);
let packageDirectory = './.embark/versions/' + packageName + '/' + version + '/';
if (fs.existsSync(packageDirectory + "/downloaded_package.tgz")) {
if (returnContent) {
let distFile = packageDirectory + returnContent;
callback(fs.readFileSync(distFile).toString());
} else {
callback(packageDirectory);
}
} else {
fs.mkdirpSync(packageDirectory);
//self.logger.info("downloading " + packageName + " " + version + "....");
download(tarball, packageDirectory + "/downloaded_package.tgz", function() {
o_fs.createReadStream(packageDirectory + '/downloaded_package.tgz').pipe(
tar.x({
strip: 1,
C: packageDirectory
}).on('end', function() {
if (returnContent) {
let distFile = packageDirectory + returnContent;
callback(fs.readFileSync(distFile).toString());
} else {
callback(packageDirectory);
}
})
);
}); });
});
}
}); });
}); });
} }

View File

@ -1,5 +1,6 @@
/*jshint esversion: 6, loopfunc: true */ /*jshint esversion: 6, loopfunc: true */
let fs = require('../core/fs.js'); let fs = require('../core/fs.js');
let async = require('async');
class Pipeline { class Pipeline {
@ -11,82 +12,105 @@ class Pipeline {
this.plugins = options.plugins; this.plugins = options.plugins;
} }
build(abi, contractsJSON, path) { build(abi, contractsJSON, path, callback) {
let self = this; let self = this;
this.buildContracts(contractsJSON); this.buildContracts(contractsJSON);
for (let targetFile in this.assetFiles) { // limit:1 due to issues when downloading required files such as web3.js
async.eachOfLimit(this.assetFiles, 1, function (files, targetFile, cb) {
let contentFiles = this.assetFiles[targetFile].map(file => { // limit:1 due to issues when downloading required files such as web3.js
self.logger.trace("reading " + file.filename); async.mapLimit(files, 1,
function(file, fileCb) {
self.logger.trace("reading " + file.filename);
let pipelinePlugins = this.plugins.getPluginsFor('pipeline'); let pipelinePlugins = self.plugins.getPluginsFor('pipeline');
if (file.filename === "$ALL_CONTRACTS") { if (file.filename === "$ALL_CONTRACTS") {
return {content: abi, filename: file.filename, path: file.path, modified: true}; return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
} else if (file.filename === "$EMBARK_JS") { } else if (file.filename === "$EMBARK_JS") {
return {content: file.content, filename: "embark.js", path: file.path, modified: true}; return file.content(function(fileContent) {
} else if (file.filename[0] === '$') { return fileCb(null, {content: fileContent, filename: "embark.js", path: file.path, modified: true});
let contractName = file.filename.substr(1);
return {content: this.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true};
} else if (file.filename === 'embark.js') {
return {content: file.content + "\n" + abi, filename: file.filename, path: file.path, modified: true};
} else if (file.filename === 'abi.js') {
return {content: abi, filename: file.filename, path: file.path, modified: true};
} else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) {
file.modified = true;
return file;
} else {
if (pipelinePlugins.length > 0) {
pipelinePlugins.forEach(function (plugin) {
try {
if (file.options && file.options.skipPipeline) {
return;
}
file.content = plugin.runPipeline({targetFile: file.filename, source: file.content});
file.modified = true;
}
catch (err) {
self.logger.error(err.message);
}
}); });
} else if (file.filename[0] === '$') {
let contractName = file.filename.substr(1);
return fileCb(null, {content: self.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true});
} else if (file.filename === 'embark.js') {
return file.content(function(fileContent) {
return fileCb(null, {content: fileContent + "\n" + abi, filename: file.filename, path: file.path, modified: true});
});
} else if (file.filename === 'abi.js') {
return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
} else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) {
file.content(function(fileContent) {
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
} else {
if (pipelinePlugins.length > 0) {
file.content(function(fileContent) {
async.eachSeries(pipelinePlugins, function(plugin, pluginCB) {
if (file.options && file.options.skipPipeline) {
return pluginCB();
}
fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent});
file.modified = true;
pluginCB();
},
function (err) {
if (err) {
self.logger.error(err.message);
}
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
});
} else {
file.content(function(fileContent) {
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
}
} }
},
function (err, contentFiles) {
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace("creating dir " + self.buildDir + dir);
fs.mkdirpSync(self.buildDir + dir);
return file; // if it's a directory
if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
contentFiles.map(function (file) {
let filename = file.filename.replace('app/', '');
filename = filename.replace(targetDir, '');
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true});
});
} else {
let content = contentFiles.map(function (file) {
if (file === undefined) {
return "";
}
return file.content;
}).join("\n");
self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
fs.writeFileSync(self.buildDir + targetFile, content);
}
cb();
} }
}); );
},
let dir = targetFile.split('/').slice(0, -1).join('/'); function (err, results) {
self.logger.trace("creating dir " + this.buildDir + dir); callback();
fs.mkdirpSync(this.buildDir + dir); });
// if it's a directory
if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
contentFiles.map(function (file) {
let filename = file.filename.replace('app/', '');
filename = filename.replace(targetDir, '');
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true});
});
} else {
let content = contentFiles.map(function (file) {
return file.content;
}).join("\n");
self.logger.info("writing file " + (this.buildDir + targetFile).bold.dim);
fs.writeFileSync(this.buildDir + targetFile, content);
}
}
} }
buildContracts(contractsJSON) { buildContracts(contractsJSON) {

View File

@ -3,6 +3,7 @@ let path = require('path');
let globule = require('globule'); let globule = require('globule');
let merge = require('merge'); let merge = require('merge');
let http = require('follow-redirects').http; let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
let shelljs = require('shelljs'); let shelljs = require('shelljs');
function joinPath() { function joinPath() {
@ -33,6 +34,10 @@ function httpGet(url, callback) {
return http.get(url, callback); return http.get(url, callback);
} }
function httpsGet(url, callback) {
return https.get(url, callback);
}
function runCmd(cmd, options) { function runCmd(cmd, options) {
let result = shelljs.exec(cmd, options || {silent: true}); let result = shelljs.exec(cmd, options || {silent: true});
if (result.code !== 0) { if (result.code !== 0) {
@ -65,6 +70,7 @@ module.exports = {
recursiveMerge: recursiveMerge, recursiveMerge: recursiveMerge,
checkIsAvailable: checkIsAvailable, checkIsAvailable: checkIsAvailable,
httpGet: httpGet, httpGet: httpGet,
httpsGet: httpsGet,
runCmd: runCmd, runCmd: runCmd,
cd: cd, cd: cd,
sed: sed, sed: sed,

View File

@ -33,6 +33,7 @@
"serve-static": "^1.11.1", "serve-static": "^1.11.1",
"shelljs": "^0.5.0", "shelljs": "^0.5.0",
"solc": "0.4.11", "solc": "0.4.11",
"tar": "^3.1.5",
"toposort": "^1.0.0", "toposort": "^1.0.0",
"underscore": "^1.8.3", "underscore": "^1.8.3",
"underscore.string": "^3.3.4", "underscore.string": "^3.3.4",

View File

@ -1,5 +1,17 @@
{ {
"default": { "default": {
"versions": {
"web3.js": "0.19.1",
"solc": "0.4.11"
},
"deployment": {
"host": "localhost",
"port": 8545
},
"dappConnection": [
"$WEB3",
"localhost:8545"
],
"gas": "auto", "gas": "auto",
"contracts": { "contracts": {
"SimpleStorage": { "SimpleStorage": {

View File

@ -9,8 +9,7 @@
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"homepage": "", "homepage": "",
"devDependencies": { "devDependencies": {},
},
"dependencies": { "dependencies": {
"embark-babel": "^1.0.0", "embark-babel": "^1.0.0",
"embark-service": "./extensions/embark-service", "embark-service": "./extensions/embark-service",