From f45ce963b4668ff834d75f1746df51533be72ee7 Mon Sep 17 00:00:00 2001 From: Iuri Matias Date: Wed, 5 Jul 2017 08:35:51 -0400 Subject: [PATCH] add file type so files are loaded when needed; support to configure web3 and solc versions --- lib/contracts/compiler.js | 27 ++++-- lib/contracts/contracts.js | 3 +- lib/contracts/solcP.js | 4 +- lib/contracts/solcW.js | 14 ++- lib/core/config.js | 54 +++++++----- lib/core/engine.js | 5 +- lib/core/file.js | 26 ++++++ lib/core/fs.js | 4 + lib/pipeline/npm.js | 68 +++++++++++---- lib/pipeline/pipeline.js | 154 +++++++++++++++++++-------------- lib/utils/utils.js | 6 ++ package.json | 1 + test_app/config/contracts.json | 12 +++ test_app/package.json | 3 +- 14 files changed, 264 insertions(+), 117 deletions(-) create mode 100644 lib/core/file.js diff --git a/lib/contracts/compiler.js b/lib/contracts/compiler.js index 05b3e072..75e7551a 100644 --- a/lib/contracts/compiler.js +++ b/lib/contracts/compiler.js @@ -6,6 +6,7 @@ class Compiler { constructor(options) { this.plugins = options.plugins; this.logger = options.logger; + this.solcVersion = options.solcVersion; } compile_contracts(contractFiles, cb) { @@ -53,16 +54,28 @@ class Compiler { let solcW; async.waterfall([ function prepareInput(callback) { - for (let i = 0; i < contractFiles.length; i++) { - // TODO: this depends on the config - let filename = contractFiles[i].filename.replace('app/contracts/', ''); - input[filename] = contractFiles[i].content.toString(); - } - callback(); + async.each(contractFiles, + function(file, fileCb) { + let filename = file.filename.replace('app/contracts/', ''); + file.content(function(fileContent) { + input[filename] = fileContent; + fileCb(); + }); + }, + function (err) { + callback(err); + } + ); + //for (let i = 0; i < contractFiles.length; i++) { + // // TODO: this depends on the config + // let filename = contractFiles[i].filename.replace('app/contracts/', ''); + // input[filename] = contractFiles[i].content.toString(); + //} + //callback(); }, function loadCompiler(callback) { // TODO: there ino need to load this twice - solcW = new SolcW(); + solcW = new SolcW(self.solcVersion); if (solcW.isCompilerLoaded()) { return callback(); } diff --git a/lib/contracts/contracts.js b/lib/contracts/contracts.js index 54fb8e89..72f9baab 100644 --- a/lib/contracts/contracts.js +++ b/lib/contracts/contracts.js @@ -12,6 +12,7 @@ class ContractsManager { this.contracts = {}; this.logger = options.logger; this.plugins = options.plugins; + this.solcVersion = options.contractsConfig.versions["solc"]; this.contractDependencies = {}; } @@ -20,7 +21,7 @@ class ContractsManager { let self = this; async.waterfall([ function compileContracts(callback) { - let compiler = new Compiler({plugins: self.plugins, logger: self.logger}); + let compiler = new Compiler({plugins: self.plugins, logger: self.logger, solcVersion: self.solcVersion}); compiler.compile_contracts(self.contractFiles, function (err, compiledObject) { self.compiledContracts = compiledObject; callback(err); diff --git a/lib/contracts/solcP.js b/lib/contracts/solcP.js index 4c903fc1..d62e8299 100644 --- a/lib/contracts/solcP.js +++ b/lib/contracts/solcP.js @@ -2,7 +2,9 @@ let solc; process.on('message', function (msg) { if (msg.action === 'loadCompiler') { - solc = require('solc'); + //solc = require('solc'); + //console.log("requiring compiler at " + msg.solcLocation); + solc = require(msg.solcLocation); process.send({result: "loadedCompiler"}); } diff --git a/lib/contracts/solcW.js b/lib/contracts/solcW.js index f18407a9..3c6c744c 100644 --- a/lib/contracts/solcW.js +++ b/lib/contracts/solcW.js @@ -1,9 +1,15 @@ let utils = require('../utils/utils.js'); let solcProcess; let compilerLoaded = false; +var npm = require('../pipeline/npm.js'); +let path = require('path'); class SolcW { + constructor(version) { + this.solcVersion = version; + } + load_compiler(done) { if (compilerLoaded) { done(); @@ -16,7 +22,13 @@ class SolcW { compilerLoaded = true; done(); }); - solcProcess.send({action: 'loadCompiler'}); + npm.getPackageVersion('solc', '0.4.10', false, function(location) { + console.log("new compiler installed at " + location); + //let requirePath = path.join(process.env.PWD, location.substr(2)); + let requirePath = path.join(process.env.PWD, location); + console.log(requirePath); + solcProcess.send({action: 'loadCompiler', solcLocation: requirePath}); + }); } isCompilerLoaded() { diff --git a/lib/core/config.js b/lib/core/config.js index 6128ab45..23224c72 100644 --- a/lib/core/config.js +++ b/lib/core/config.js @@ -1,4 +1,5 @@ var fs = require('./fs.js'); +var File = require('./file.js'); var Plugins = require('./plugins.js'); var utils = require('../utils/utils.js'); var npm = require('../pipeline/npm.js'); @@ -45,9 +46,9 @@ Config.prototype.loadConfigFiles = function(options) { this.loadStorageConfigFile(); this.loadCommunicationConfigFile(); + this.loadContractsConfigFile(); this.loadPipelineConfigFile(); - this.loadContractsConfigFile(); this.loadWebServerConfigFile(); this.loadChainTrackerFile(); this.loadPluginContractFiles(); @@ -58,8 +59,8 @@ Config.prototype.reloadConfig = function() { this.loadBlockchainConfigFile(); this.loadStorageConfigFile(); this.loadCommunicationConfigFile(); - this.loadPipelineConfigFile(); this.loadContractsConfigFile(); + this.loadPipelineConfigFile(); this.loadChainTrackerFile(); }; @@ -234,34 +235,38 @@ Config.prototype.loadFiles = function(files) { if (file === 'embark.js') { if (self.blockchainConfig.enabled || self.communicationConfig.provider === 'whisper' || self.communicationConfig.available_providers.indexOf('whisper') >= 0) { - readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")}); + let web3Version = self.contractsConfig.versions["web3.js"]; + if (web3Version) { + //if (false) { + //readFiles.push(new File({filename: 'web3-' + web3Version + '.js', type: 'custom', resolver: function(callback) { + readFiles.push(new File({filename: 'web3.js', type: 'custom', resolver: function(callback) { + npm.getPackageVersion('web3', web3Version, 'dist/web3.js', function(web3Content) { + callback(web3Content); + }); + }})); + } else { + readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"})); + } } if (self.storageConfig.enabled && (self.storageConfig.provider === 'ipfs' || self.storageConfig.available_providers.indexOf('ipfs') >= 0)) { - readFiles.push({filename: 'ipfs.js', content: fs.readFileSync(fs.embarkPath("js/ipfs.js")).toString(), path: fs.embarkPath("js/ipfs.js")}); + readFiles.push(new File({filename: 'ipfs.js', type: 'embark_internal', path: "js/ipfs.js"})); } if (self.communicationConfig.enabled && (self.communicationConfig.provider === 'orbit' || self.communicationConfig.available_providers.indexOf('orbit') >= 0)) { // TODO: remove duplicated files if functionality is the same for storage and orbit - readFiles.push({filename: 'ipfs-api.js', content: fs.readFileSync(fs.embarkPath("js/ipfs-api.min.js")).toString(), path: fs.embarkPath("js/ipfs-api.min.js")}); - readFiles.push({filename: 'orbit.js', content: fs.readFileSync(fs.embarkPath("js/orbit.min.js")).toString(), path: fs.embarkPath("js/orbit.min.js")}); + readFiles.push(new File({filename: 'ipfs-api.js', type: 'embark_internal', path: "js/ipfs-api.min.js"})); + readFiles.push(new File({filename: 'orbit.js', type: 'embark_internal', path: "js/orbit.min.js"})); } - readFiles.push({filename: 'embark.js', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")}); + readFiles.push(new File({filename: 'embark.js', type: 'embark_internal', path: "js/build/embark.bundle.js"})); } if (file === '$EMBARK_JS') { - readFiles.push({filename: '$EMBARK_JS', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")}); - } - if (file.indexOf("web3-") === 0) { - let web3Version = file.split('web3-')[1].split(".js")[0]; - npm.getPackageVersion('web3', web3Version, function(web3Content) { - self.logger.error('downloaded web3'); - readFiles.push({filename: file, content: web3Content, path: fs.embarkPath("js/web3.js")}); - }); - } - if (file === "web3.js") { - readFiles.push({filename: 'web3.js', content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")}); + readFiles.push(new File({filename: '$EMBARK_JS', type: 'embark_internal', path: "js/build/embark.bundle.js"})); } + //if (file === "web3.js") { + // readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js")); + //} }); // get plugins @@ -298,13 +303,13 @@ Config.prototype.loadFiles = function(files) { return; } else if (file.indexOf("web3") === 0) { return; - } else if (file === 'abi.js') { - readFiles.push({filename: file, content: "", path: file}); + //} else if (file === 'abi.js') { + // readFiles.push({filename: file, content: "", path: file}); } else { if (file.indexOf('.') >= 0) { - readFiles.push({filename: file, content: fs.readFileSync(file).toString(), path: file}); + readFiles.push(new File({filename: file, type: "dapp_file", path: file})); } else if (file[0] === '$') { - readFiles.push({filename: file, content: "", path: file}); + //readFiles.push(new File({filename: file, type: 'embark_internal', path: file})); } } }); @@ -320,7 +325,10 @@ Config.prototype.loadPluginContractFiles = function() { contractsPlugins.forEach(function(plugin) { plugin.contractsFiles.forEach(function(file) { var filename = file.replace('./',''); - self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)}); + //self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)}); + self.contractsFiles.push(new File({filename: filename, type: 'custom', resolver: function(callback) { + callback(plugin.loadPluginFile(file)); + }})); }); }); } diff --git a/lib/core/engine.js b/lib/core/engine.js index 47af0ad8..31d48e2d 100644 --- a/lib/core/engine.js +++ b/lib/core/engine.js @@ -83,8 +83,9 @@ class Engine { this.events.on('abi', function (abi, contractsJSON) { self.currentAbi = abi; self.contractsJSON = contractsJSON; - pipeline.build(abi, contractsJSON); - self.events.emit('outputDone'); + pipeline.build(abi, contractsJSON, null, function() { + self.events.emit('outputDone'); + }); }); // TODO: still need to redeploy contracts because the original contracts // config is being corrupted diff --git a/lib/core/file.js b/lib/core/file.js new file mode 100644 index 00000000..adce6ab0 --- /dev/null +++ b/lib/core/file.js @@ -0,0 +1,26 @@ +let fs = require('./fs.js'); + +class File { + + constructor(options) { + this.filename = options.filename; + this.type = options.type; + this.path = options.path; + this.resolver = options.resolver; + } + + content(callback) { + if (this.type === 'embark_internal') { + return callback(fs.readFileSync(fs.embarkPath(this.path)).toString()); + } else if (this.type === 'dapp_file') { + return callback(fs.readFileSync(this.path).toString()); + } else if (this.type === 'custom') { + return this.resolver(callback); + } else { + throw new Error("unknown file: " + this.filename); + } + } + +} + +module.exports = File; diff --git a/lib/core/fs.js b/lib/core/fs.js index 8b3be66a..d61933fa 100644 --- a/lib/core/fs.js +++ b/lib/core/fs.js @@ -34,6 +34,10 @@ function embarkPath(fileOrDir) { return utils.joinPath(__dirname, '/../../', fileOrDir); } +function dappPath() { + return process.env.PWD; +} + module.exports = { mkdirpSync: mkdirpSync, copySync: copySync, diff --git a/lib/pipeline/npm.js b/lib/pipeline/npm.js index 59b725dc..cd9367b3 100644 --- a/lib/pipeline/npm.js +++ b/lib/pipeline/npm.js @@ -1,12 +1,16 @@ let utils = require('../utils/utils.js'); +let fs = require('../core/fs.js'); +let o_fs = require('fs-extra'); + +let http = require('follow-redirects').http; +let https = require('follow-redirects').https; +var tar = require('tar'); module.exports = { - // TODO: need to refactor to make it truly generalistic, perhaps by - // downloading the tarball itself - getPackageVersion: function(packageName, version, callback) { - let npmRegistry = "http://registry.npmjs.org/" + packageName + "/" + version; + getPackageVersion: function(packageName, version, returnContent, callback) { + let npmRegistry = "https://registry.npmjs.org/" + packageName + "/" + version; - utils.httpGet(npmRegistry, function (res) { + utils.httpsGet(npmRegistry, function (res) { let body = ''; res.on('data', function (d) { @@ -14,19 +18,53 @@ module.exports = { }); res.on('end', function () { let registryJSON = JSON.parse(body); - let gitHash = registryJSON.gitHead; - let repo = registryJSON.homepage.split("github.com/")[1]; - let gitUrl = "http://raw.githubusercontent.com/" + repo + "/" + gitHash + "/dist/" + packageName + ".js"; - utils.httpGet(gitUrl, function (res2) { - let body = ''; - res2.on('data', function (d) { - body += d; + let tarball = registryJSON.dist.tarball; + + var download = function(url, dest, cb) { + var file = o_fs.createWriteStream(dest); + var request = (url.substring(0,5) === 'https' ? https : http).get(url, function(response) { + response.pipe(file); + file.on('finish', function() { + file.close(cb); // close() is async, call cb after close completes. + }); + }).on('error', function(err) { // Handle errors + fs.unlink(dest); // Delete the file async. (But we don't check the result) + if (cb) cb(err.message); }); - res2.on('end', function () { - callback(body); + }; + + let packageDirectory = './.embark/versions/' + packageName + '/' + version + '/'; + + if (fs.existsSync(packageDirectory + "/downloaded_package.tgz")) { + if (returnContent) { + let distFile = packageDirectory + returnContent; + callback(fs.readFileSync(distFile).toString()); + } else { + callback(packageDirectory); + } + } else { + fs.mkdirpSync(packageDirectory); + //self.logger.info("downloading " + packageName + " " + version + "...."); + + download(tarball, packageDirectory + "/downloaded_package.tgz", function() { + o_fs.createReadStream(packageDirectory + '/downloaded_package.tgz').pipe( + tar.x({ + strip: 1, + C: packageDirectory + }).on('end', function() { + if (returnContent) { + let distFile = packageDirectory + returnContent; + callback(fs.readFileSync(distFile).toString()); + } else { + callback(packageDirectory); + } + }) + ); }); - }); + + } + }); }); } diff --git a/lib/pipeline/pipeline.js b/lib/pipeline/pipeline.js index cc38832d..ecc99978 100644 --- a/lib/pipeline/pipeline.js +++ b/lib/pipeline/pipeline.js @@ -1,5 +1,6 @@ /*jshint esversion: 6, loopfunc: true */ let fs = require('../core/fs.js'); +let async = require('async'); class Pipeline { @@ -11,82 +12,105 @@ class Pipeline { this.plugins = options.plugins; } - build(abi, contractsJSON, path) { + build(abi, contractsJSON, path, callback) { let self = this; this.buildContracts(contractsJSON); - for (let targetFile in this.assetFiles) { + // limit:1 due to issues when downloading required files such as web3.js + async.eachOfLimit(this.assetFiles, 1, function (files, targetFile, cb) { - let contentFiles = this.assetFiles[targetFile].map(file => { - self.logger.trace("reading " + file.filename); + // limit:1 due to issues when downloading required files such as web3.js + async.mapLimit(files, 1, + function(file, fileCb) { + self.logger.trace("reading " + file.filename); - let pipelinePlugins = this.plugins.getPluginsFor('pipeline'); + let pipelinePlugins = self.plugins.getPluginsFor('pipeline'); - if (file.filename === "$ALL_CONTRACTS") { - return {content: abi, filename: file.filename, path: file.path, modified: true}; - } else if (file.filename === "$EMBARK_JS") { - return {content: file.content, filename: "embark.js", path: file.path, modified: true}; - } else if (file.filename[0] === '$') { - let contractName = file.filename.substr(1); - return {content: this.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true}; - } else if (file.filename === 'embark.js') { - return {content: file.content + "\n" + abi, filename: file.filename, path: file.path, modified: true}; - } else if (file.filename === 'abi.js') { - return {content: abi, filename: file.filename, path: file.path, modified: true}; - } else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) { - file.modified = true; - return file; - } else { - - if (pipelinePlugins.length > 0) { - pipelinePlugins.forEach(function (plugin) { - try { - if (file.options && file.options.skipPipeline) { - return; - } - file.content = plugin.runPipeline({targetFile: file.filename, source: file.content}); - file.modified = true; - } - catch (err) { - self.logger.error(err.message); - } + if (file.filename === "$ALL_CONTRACTS") { + return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true}); + } else if (file.filename === "$EMBARK_JS") { + return file.content(function(fileContent) { + return fileCb(null, {content: fileContent, filename: "embark.js", path: file.path, modified: true}); }); + } else if (file.filename[0] === '$') { + let contractName = file.filename.substr(1); + return fileCb(null, {content: self.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true}); + } else if (file.filename === 'embark.js') { + return file.content(function(fileContent) { + return fileCb(null, {content: fileContent + "\n" + abi, filename: file.filename, path: file.path, modified: true}); + }); + } else if (file.filename === 'abi.js') { + return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true}); + } else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) { + file.content(function(fileContent) { + return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true}); + }); + } else { + + if (pipelinePlugins.length > 0) { + file.content(function(fileContent) { + async.eachSeries(pipelinePlugins, function(plugin, pluginCB) { + if (file.options && file.options.skipPipeline) { + return pluginCB(); + } + + fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent}); + file.modified = true; + pluginCB(); + }, + function (err) { + if (err) { + self.logger.error(err.message); + } + return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true}); + }); + }); + } else { + file.content(function(fileContent) { + return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true}); + }); + } } + }, + function (err, contentFiles) { + let dir = targetFile.split('/').slice(0, -1).join('/'); + self.logger.trace("creating dir " + self.buildDir + dir); + fs.mkdirpSync(self.buildDir + dir); - return file; + // if it's a directory + if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) { + let targetDir = targetFile; + + if (targetDir.slice(-1) !== '/') { + targetDir = targetDir + '/'; + } + + contentFiles.map(function (file) { + let filename = file.filename.replace('app/', ''); + filename = filename.replace(targetDir, ''); + self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim); + + fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true}); + }); + } else { + let content = contentFiles.map(function (file) { + if (file === undefined) { + return ""; + } + return file.content; + }).join("\n"); + + self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim); + fs.writeFileSync(self.buildDir + targetFile, content); + } + cb(); } - }); - - let dir = targetFile.split('/').slice(0, -1).join('/'); - self.logger.trace("creating dir " + this.buildDir + dir); - fs.mkdirpSync(this.buildDir + dir); - - // if it's a directory - if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) { - let targetDir = targetFile; - - if (targetDir.slice(-1) !== '/') { - targetDir = targetDir + '/'; - } - - contentFiles.map(function (file) { - let filename = file.filename.replace('app/', ''); - filename = filename.replace(targetDir, ''); - self.logger.info("writing file " + (self.buildDir + targetDir + filename).bold.dim); - - fs.copySync(self.buildDir + targetDir + filename, file.path, {overwrite: true}); - }); - } else { - let content = contentFiles.map(function (file) { - return file.content; - }).join("\n"); - - self.logger.info("writing file " + (this.buildDir + targetFile).bold.dim); - fs.writeFileSync(this.buildDir + targetFile, content); - } - } - + ); + }, + function (err, results) { + callback(); + }); } buildContracts(contractsJSON) { diff --git a/lib/utils/utils.js b/lib/utils/utils.js index 354e9508..39d66bee 100644 --- a/lib/utils/utils.js +++ b/lib/utils/utils.js @@ -3,6 +3,7 @@ let path = require('path'); let globule = require('globule'); let merge = require('merge'); let http = require('follow-redirects').http; +let https = require('follow-redirects').https; let shelljs = require('shelljs'); function joinPath() { @@ -33,6 +34,10 @@ function httpGet(url, callback) { return http.get(url, callback); } +function httpsGet(url, callback) { + return https.get(url, callback); +} + function runCmd(cmd, options) { let result = shelljs.exec(cmd, options || {silent: true}); if (result.code !== 0) { @@ -65,6 +70,7 @@ module.exports = { recursiveMerge: recursiveMerge, checkIsAvailable: checkIsAvailable, httpGet: httpGet, + httpsGet: httpsGet, runCmd: runCmd, cd: cd, sed: sed, diff --git a/package.json b/package.json index a1062f9c..7f20bc07 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "serve-static": "^1.11.1", "shelljs": "^0.5.0", "solc": "0.4.11", + "tar": "^3.1.5", "toposort": "^1.0.0", "underscore": "^1.8.3", "underscore.string": "^3.3.4", diff --git a/test_app/config/contracts.json b/test_app/config/contracts.json index 7149a00b..265d20f4 100644 --- a/test_app/config/contracts.json +++ b/test_app/config/contracts.json @@ -1,5 +1,17 @@ { "default": { + "versions": { + "web3.js": "0.19.1", + "solc": "0.4.11" + }, + "deployment": { + "host": "localhost", + "port": 8545 + }, + "dappConnection": [ + "$WEB3", + "localhost:8545" + ], "gas": "auto", "contracts": { "SimpleStorage": { diff --git a/test_app/package.json b/test_app/package.json index b330dfbf..ebec0043 100644 --- a/test_app/package.json +++ b/test_app/package.json @@ -9,8 +9,7 @@ "author": "", "license": "ISC", "homepage": "", - "devDependencies": { - }, + "devDependencies": {}, "dependencies": { "embark-babel": "^1.0.0", "embark-service": "./extensions/embark-service",