status-im
/
embark-area-51
mirror of https://github.com/status-im/embark-area-51.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

add file type so files are loaded when needed; support to configure web3 and solc versions

This commit is contained in:
Iuri Matias 2017-07-05 08:35:51 -04:00
parent 9b98980efd
commit f45ce963b4
14 changed files with 264 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/contracts/compiler.js
View File

@ -6,6 +6,7 @@ class Compiler {
constructor(options) {
this.plugins = options.plugins;
this.logger = options.logger;
this.solcVersion = options.solcVersion;
}
compile_contracts(contractFiles, cb) {
@ -53,16 +54,28 @@ class Compiler {
let solcW;
async.waterfall([
function prepareInput(callback) {
for (let i = 0; i < contractFiles.length; i++) {
// TODO: this depends on the config
let filename = contractFiles[i].filename.replace('app/contracts/', '');
input[filename] = contractFiles[i].content.toString();
}
callback();
async.each(contractFiles,
function(file, fileCb) {
let filename = file.filename.replace('app/contracts/', '');
file.content(function(fileContent) {
input[filename] = fileContent;
fileCb();
});
},
function (err) {
callback(err);
}
);
//for (let i = 0; i < contractFiles.length; i++) {
// // TODO: this depends on the config
// let filename = contractFiles[i].filename.replace('app/contracts/', '');
// input[filename] = contractFiles[i].content.toString();
//}
//callback();
},
function loadCompiler(callback) {
// TODO: there ino need to load this twice
solcW = new SolcW();
solcW = new SolcW(self.solcVersion);
if (solcW.isCompilerLoaded()) {
return callback();
}

3
lib/contracts/contracts.js
View File

@ -12,6 +12,7 @@ class ContractsManager {
this.contracts = {};
this.logger = options.logger;
this.plugins = options.plugins;
this.solcVersion = options.contractsConfig.versions["solc"];
this.contractDependencies = {};
}
@ -20,7 +21,7 @@ class ContractsManager {
let self = this;
async.waterfall([
function compileContracts(callback) {
let compiler = new Compiler({plugins: self.plugins, logger: self.logger});
let compiler = new Compiler({plugins: self.plugins, logger: self.logger, solcVersion: self.solcVersion});
compiler.compile_contracts(self.contractFiles, function (err, compiledObject) {
self.compiledContracts = compiledObject;
callback(err);

4
lib/contracts/solcP.js
View File

@ -2,7 +2,9 @@ let solc;
process.on('message', function (msg) {
if (msg.action === 'loadCompiler') {
solc = require('solc');
//solc = require('solc');
//console.log("requiring compiler at " + msg.solcLocation);
solc = require(msg.solcLocation);
process.send({result: "loadedCompiler"});
}

14
lib/contracts/solcW.js
View File

@ -1,9 +1,15 @@
let utils = require('../utils/utils.js');
let solcProcess;
let compilerLoaded = false;
var npm = require('../pipeline/npm.js');
let path = require('path');
class SolcW {
constructor(version) {
this.solcVersion = version;
}
load_compiler(done) {
if (compilerLoaded) {
done();
@ -16,7 +22,13 @@ class SolcW {
compilerLoaded = true;
done();
});
solcProcess.send({action: 'loadCompiler'});
npm.getPackageVersion('solc', '0.4.10', false, function(location) {
console.log("new compiler installed at " + location);
//let requirePath = path.join(process.env.PWD, location.substr(2));
let requirePath = path.join(process.env.PWD, location);
console.log(requirePath);
solcProcess.send({action: 'loadCompiler', solcLocation: requirePath});
});
}
isCompilerLoaded() {

54
lib/core/config.js
View File

@ -1,4 +1,5 @@
var fs = require('./fs.js');
var File = require('./file.js');
var Plugins = require('./plugins.js');
var utils = require('../utils/utils.js');
var npm = require('../pipeline/npm.js');
@ -45,9 +46,9 @@ Config.prototype.loadConfigFiles = function(options) {
this.loadStorageConfigFile();
this.loadCommunicationConfigFile();
this.loadContractsConfigFile();
this.loadPipelineConfigFile();
this.loadContractsConfigFile();
this.loadWebServerConfigFile();
this.loadChainTrackerFile();
this.loadPluginContractFiles();
@ -58,8 +59,8 @@ Config.prototype.reloadConfig = function() {
this.loadBlockchainConfigFile();
this.loadStorageConfigFile();
this.loadCommunicationConfigFile();
this.loadPipelineConfigFile();
this.loadContractsConfigFile();
this.loadPipelineConfigFile();
this.loadChainTrackerFile();
};
@ -234,34 +235,38 @@ Config.prototype.loadFiles = function(files) {
if (file === 'embark.js') {
if (self.blockchainConfig.enabled || self.communicationConfig.provider === 'whisper' || self.communicationConfig.available_providers.indexOf('whisper') >= 0) {
readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")});
let web3Version = self.contractsConfig.versions["web3.js"];
if (web3Version) {
//if (false) {
//readFiles.push(new File({filename: 'web3-' + web3Version + '.js', type: 'custom', resolver: function(callback) {
readFiles.push(new File({filename: 'web3.js', type: 'custom', resolver: function(callback) {
npm.getPackageVersion('web3', web3Version, 'dist/web3.js', function(web3Content) {
callback(web3Content);
});
}}));
} else {
readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"}));
}
}
if (self.storageConfig.enabled && (self.storageConfig.provider === 'ipfs' || self.storageConfig.available_providers.indexOf('ipfs') >= 0)) {
readFiles.push({filename: 'ipfs.js', content: fs.readFileSync(fs.embarkPath("js/ipfs.js")).toString(), path: fs.embarkPath("js/ipfs.js")});
readFiles.push(new File({filename: 'ipfs.js', type: 'embark_internal', path: "js/ipfs.js"}));
}
if (self.communicationConfig.enabled && (self.communicationConfig.provider === 'orbit' || self.communicationConfig.available_providers.indexOf('orbit') >= 0)) {
// TODO: remove duplicated files if functionality is the same for storage and orbit
readFiles.push({filename: 'ipfs-api.js', content: fs.readFileSync(fs.embarkPath("js/ipfs-api.min.js")).toString(), path: fs.embarkPath("js/ipfs-api.min.js")});
readFiles.push({filename: 'orbit.js', content: fs.readFileSync(fs.embarkPath("js/orbit.min.js")).toString(), path: fs.embarkPath("js/orbit.min.js")});
readFiles.push(new File({filename: 'ipfs-api.js', type: 'embark_internal', path: "js/ipfs-api.min.js"}));
readFiles.push(new File({filename: 'orbit.js', type: 'embark_internal', path: "js/orbit.min.js"}));
}
readFiles.push({filename: 'embark.js', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")});
readFiles.push(new File({filename: 'embark.js', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
}
if (file === '$EMBARK_JS') {
readFiles.push({filename: '$EMBARK_JS', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")});
}
if (file.indexOf("web3-") === 0) {
let web3Version = file.split('web3-')[1].split(".js")[0];
npm.getPackageVersion('web3', web3Version, function(web3Content) {
self.logger.error('downloaded web3');
readFiles.push({filename: file, content: web3Content, path: fs.embarkPath("js/web3.js")});
});
}
if (file === "web3.js") {
readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")});
readFiles.push(new File({filename: '$EMBARK_JS', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
}
//if (file === "web3.js") {
// readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"));
//}
});
// get plugins
@ -298,13 +303,13 @@ Config.prototype.loadFiles = function(files) {
return;
} else if (file.indexOf("web3") === 0) {
return;
} else if (file === 'abi.js') {
readFiles.push({filename: file, content: "", path: file});
//} else if (file === 'abi.js') {
// readFiles.push({filename: file, content: "", path: file});
} else {
if (file.indexOf('.') >= 0) {
readFiles.push({filename: file, content: fs.readFileSync(file).toString(), path: file});
readFiles.push(new File({filename: file, type: "dapp_file", path: file}));
} else if (file[0] === '$') {
readFiles.push({filename: file, content: "", path: file});
//readFiles.push(new File({filename: file, type: 'embark_internal', path: file}));
}
}
});
@ -320,7 +325,10 @@ Config.prototype.loadPluginContractFiles = function() {
contractsPlugins.forEach(function(plugin) {
plugin.contractsFiles.forEach(function(file) {
var filename = file.replace('./','');
self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)});
//self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)});
self.contractsFiles.push(new File({filename: filename, type: 'custom', resolver: function(callback) {
callback(plugin.loadPluginFile(file));
}}));
});
});
}

5
lib/core/engine.js
View File

@ -83,8 +83,9 @@ class Engine {
this.events.on('abi', function (abi, contractsJSON) {
self.currentAbi = abi;
self.contractsJSON = contractsJSON;
pipeline.build(abi, contractsJSON);
self.events.emit('outputDone');
pipeline.build(abi, contractsJSON, null, function() {
self.events.emit('outputDone');
});
});
// TODO: still need to redeploy contracts because the original contracts
// config is being corrupted

26
lib/core/file.js Normal file
View File

@ -0,0 +1,26 @@
let fs = require('./fs.js');
class File {
constructor(options) {
this.filename = options.filename;
this.type = options.type;
this.path = options.path;
this.resolver = options.resolver;
}
content(callback) {
if (this.type === 'embark_internal') {
return callback(fs.readFileSync(fs.embarkPath(this.path)).toString());
} else if (this.type === 'dapp_file') {
return callback(fs.readFileSync(this.path).toString());
} else if (this.type === 'custom') {
return this.resolver(callback);
} else {
throw new Error("unknown file: " + this.filename);
}
}
}
module.exports = File;

4
lib/core/fs.js
View File

@ -34,6 +34,10 @@ function embarkPath(fileOrDir) {
return utils.joinPath(__dirname, '/../../', fileOrDir);
}
function dappPath() {
return process.env.PWD;
}
module.exports = {
mkdirpSync: mkdirpSync,
copySync: copySync,

68
lib/pipeline/npm.js
View File

@ -1,12 +1,16 @@
let utils = require('../utils/utils.js');
let fs = require('../core/fs.js');
let o_fs = require('fs-extra');
let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
var tar = require('tar');
module.exports = {
// TODO: need to refactor to make it truly generalistic, perhaps by
// downloading the tarball itself
getPackageVersion: function(packageName, version, callback) {
let npmRegistry = "http://registry.npmjs.org/" + packageName + "/" + version;
getPackageVersion: function(packageName, version, returnContent, callback) {
let npmRegistry = "https://registry.npmjs.org/" + packageName + "/" + version;
utils.httpGet(npmRegistry, function (res) {
utils.httpsGet(npmRegistry, function (res) {
let body = '';
res.on('data', function (d) {
@ -14,19 +18,53 @@ module.exports = {
});
res.on('end', function () {
let registryJSON = JSON.parse(body);
let gitHash = registryJSON.gitHead;
let repo = registryJSON.homepage.split("github.com/")[1];
let gitUrl = "http://raw.githubusercontent.com/" + repo + "/" + gitHash + "/dist/" + packageName + ".js";
utils.httpGet(gitUrl, function (res2) {
let body = '';
res2.on('data', function (d) {
body += d;
let tarball = registryJSON.dist.tarball;
var download = function(url, dest, cb) {
var file = o_fs.createWriteStream(dest);
var request = (url.substring(0,5) === 'https' ? https : http).get(url, function(response) {
response.pipe(file);
file.on('finish', function() {
file.close(cb); // close() is async, call cb after close completes.
});
}).on('error', function(err) { // Handle errors
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
});
res2.on('end', function () {
callback(body);
};
let packageDirectory = './.embark/versions/' + packageName + '/' + version + '/';
if (fs.existsSync(packageDirectory + "/downloaded_package.tgz")) {
if (returnContent) {
let distFile = packageDirectory + returnContent;
callback(fs.readFileSync(distFile).toString());
} else {
callback(packageDirectory);
}
} else {
fs.mkdirpSync(packageDirectory);
//self.logger.info("downloading " + packageName + " " + version + "....");
download(tarball, packageDirectory + "/downloaded_package.tgz", function() {
o_fs.createReadStream(packageDirectory + '/downloaded_package.tgz').pipe(
tar.x({
strip: 1,
C: packageDirectory
}).on('end', function() {
if (returnContent) {
let distFile = packageDirectory + returnContent;
callback(fs.readFileSync(distFile).toString());
} else {
callback(packageDirectory);
}
})
);
});
});
}
});
});
}

154
lib/pipeline/pipeline.js
View File

@ -1,5 +1,6 @@
/*jshint esversion: 6, loopfunc: true */
let fs = require('../core/fs.js');
let async = require('async');
class Pipeline {
@ -11,82 +12,105 @@ class Pipeline {
this.plugins = options.plugins;
}
build(abi, contractsJSON, path) {
build(abi, contractsJSON, path, callback) {
let self = this;
this.buildContracts(contractsJSON);
for (let targetFile in this.assetFiles) {
// limit:1 due to issues when downloading required files such as web3.js
async.eachOfLimit(this.assetFiles, 1, function (files, targetFile, cb) {
let contentFiles = this.assetFiles[targetFile].map(file => {
self.logger.trace("reading " + file.filename);
// limit:1 due to issues when downloading required files such as web3.js
async.mapLimit(files, 1,
function(file, fileCb) {
self.logger.trace("reading " + file.filename);
let pipelinePlugins = this.plugins.getPluginsFor('pipeline');
let pipelinePlugins = self.plugins.getPluginsFor('pipeline');
if (file.filename === "$ALL_CONTRACTS") {
return {content: abi, filename: file.filename, path: file.path, modified: true};
} else if (file.filename === "$EMBARK_JS") {
return {content: file.content, filename: "embark.js", path: file.path, modified: true};
} else if (file.filename[0] === '$') {
let contractName = file.filename.substr(1);
return {content: this.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true};
} else if (file.filename === 'embark.js') {
return {content: file.content + "\n" + abi, filename: file.filename, path: file.path, modified: true};
} else if (file.filename === 'abi.js') {
return {content: abi, filename: file.filename, path: file.path, modified: true};
} else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) {
file.modified = true;
return file;
} else {
if (pipelinePlugins.length > 0) {
pipelinePlugins.forEach(function (plugin) {
try {
if (file.options && file.options.skipPipeline) {
return;
}
file.content = plugin.runPipeline({targetFile: file.filename, source: file.content});
file.modified = true;
}
catch (err) {
self.logger.error(err.message);
}
if (file.filename === "$ALL_CONTRACTS") {
return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
} else if (file.filename === "$EMBARK_JS") {
return file.content(function(fileContent) {
return fileCb(null, {content: fileContent, filename: "embark.js", path: file.path, modified: true});
});
} else if (file.filename[0] === '$') {
let contractName = file.filename.substr(1);
return fileCb(null, {content: self.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true});
} else if (file.filename === 'embark.js') {
return file.content(function(fileContent) {
return fileCb(null, {content: fileContent + "\n" + abi, filename: file.filename, path: file.path, modified: true});
});
} else if (file.filename === 'abi.js') {
return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
} else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) {
file.content(function(fileContent) {
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
} else {
if (pipelinePlugins.length > 0) {
file.content(function(fileContent) {
async.eachSeries(pipelinePlugins, function(plugin, pluginCB) {
if (file.options && file.options.skipPipeline) {
return pluginCB();
}
fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent});
file.modified = true;
pluginCB();
},
function (err) {
if (err) {
self.logger.error(err.message);
}
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
});
} else {
file.content(function(fileContent) {
return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
});
}
}
},
function (err, contentFiles) {
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace("creating dir " + self.buildDir + dir);
fs.mkdirpSync(self.buildDir + dir);
return file;
// if it's a directory
if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
contentFiles.map(function (file) {
let filename = file.filename.replace('app/', '');
filename = filename.replace(targetDir, '');
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true});
});
} else {
let content = contentFiles.map(function (file) {
if (file === undefined) {
return "";
}
return file.content;
}).join("\n");
self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
fs.writeFileSync(self.buildDir + targetFile, content);
}
cb();
}
});
let dir = targetFile.split('/').slice(0, -1).join('/');
self.logger.trace("creating dir " + this.buildDir + dir);
fs.mkdirpSync(this.buildDir + dir);
// if it's a directory
if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
let targetDir = targetFile;
if (targetDir.slice(-1) !== '/') {
targetDir = targetDir + '/';
}
contentFiles.map(function (file) {
let filename = file.filename.replace('app/', '');
filename = filename.replace(targetDir, '');
self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim);
fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true});
});
} else {
let content = contentFiles.map(function (file) {
return file.content;
}).join("\n");
self.logger.info("writing file " + (this.buildDir + targetFile).bold.dim);
fs.writeFileSync(this.buildDir + targetFile, content);
}
}
);
},
function (err, results) {
callback();
});
}
buildContracts(contractsJSON) {

6
lib/utils/utils.js
View File

@ -3,6 +3,7 @@ let path = require('path');
let globule = require('globule');
let merge = require('merge');
let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
let shelljs = require('shelljs');
function joinPath() {
@ -33,6 +34,10 @@ function httpGet(url, callback) {
return http.get(url, callback);
}
function httpsGet(url, callback) {
return https.get(url, callback);
}
function runCmd(cmd, options) {
let result = shelljs.exec(cmd, options || {silent: true});
if (result.code !== 0) {
@ -65,6 +70,7 @@ module.exports = {
recursiveMerge: recursiveMerge,
checkIsAvailable: checkIsAvailable,
httpGet: httpGet,
httpsGet: httpsGet,
runCmd: runCmd,
cd: cd,
sed: sed,

1
package.json
View File

@ -33,6 +33,7 @@
"serve-static": "^1.11.1",
"shelljs": "^0.5.0",
"solc": "0.4.11",
"tar": "^3.1.5",
"toposort": "^1.0.0",
"underscore": "^1.8.3",
"underscore.string": "^3.3.4",

12
test_app/config/contracts.json
View File

@ -1,5 +1,17 @@
{
"default": {
"versions": {
"web3.js": "0.19.1",
"solc": "0.4.11"
},
"deployment": {
"host": "localhost",
"port": 8545
},
"dappConnection": [
"$WEB3",
"localhost:8545"
],
"gas": "auto",
"contracts": {
"SimpleStorage": {

3
test_app/package.json
View File

@ -9,8 +9,7 @@
"author": "",
"license": "ISC",
"homepage": "",
"devDependencies": {
},
"devDependencies": {},
"dependencies": {
"embark-babel": "^1.0.0",
"embark-service": "./extensions/embark-service",