add file type so files are loaded when needed; support to configure web3 and solc versions

lib/contracts/compiler.js

@@ -6,6 +6,7 @@ class Compiler {
   constructor(options) {
     this.plugins = options.plugins;
     this.logger = options.logger;
+    this.solcVersion = options.solcVersion;
   }
 
   compile_contracts(contractFiles, cb) {
@@ -53,16 +54,28 @@ class Compiler {
     let solcW;
     async.waterfall([
       function prepareInput(callback) {
-        for (let i = 0; i < contractFiles.length; i++) {
+        async.each(contractFiles,
-          // TODO: this depends on the config
+                   function(file, fileCb) {
-          let filename = contractFiles[i].filename.replace('app/contracts/', '');
+                     let filename = file.filename.replace('app/contracts/', '');
-          input[filename] = contractFiles[i].content.toString();
+                     file.content(function(fileContent) {
+                      input[filename] = fileContent;
+                      fileCb();
+                     });
+                   },
+                   function (err) {
+                     callback(err);
                    }
-        callback();
+        );
+        //for (let i = 0; i < contractFiles.length; i++) {
+        //  // TODO: this depends on the config
+        //  let filename = contractFiles[i].filename.replace('app/contracts/', '');
+        //  input[filename] = contractFiles[i].content.toString();
+        //}
+        //callback();
       },
       function loadCompiler(callback) {
         // TODO: there ino need to load this twice
-        solcW = new SolcW();
+        solcW = new SolcW(self.solcVersion);
         if (solcW.isCompilerLoaded()) {
           return callback();
         }

lib/contracts/contracts.js

@@ -12,6 +12,7 @@ class ContractsManager {
     this.contracts = {};
     this.logger = options.logger;
     this.plugins = options.plugins;
+    this.solcVersion = options.contractsConfig.versions["solc"];
 
     this.contractDependencies = {};
   }
@@ -20,7 +21,7 @@ class ContractsManager {
     let self = this;
     async.waterfall([
       function compileContracts(callback) {
-        let compiler = new Compiler({plugins: self.plugins, logger: self.logger});
+        let compiler = new Compiler({plugins: self.plugins, logger: self.logger, solcVersion: self.solcVersion});
         compiler.compile_contracts(self.contractFiles, function (err, compiledObject) {
           self.compiledContracts = compiledObject;
           callback(err);

lib/contracts/solcP.js

@@ -2,7 +2,9 @@ let solc;
 
 process.on('message', function (msg) {
   if (msg.action === 'loadCompiler') {
-    solc = require('solc');
+    //solc = require('solc');
+    //console.log("requiring compiler at " + msg.solcLocation);
+    solc = require(msg.solcLocation);
     process.send({result: "loadedCompiler"});
   }
 

lib/contracts/solcW.js

@@ -1,9 +1,15 @@
 let utils = require('../utils/utils.js');
 let solcProcess;
 let compilerLoaded = false;
+var npm = require('../pipeline/npm.js');
+let path = require('path');
 
 class SolcW {
 
+  constructor(version) {
+    this.solcVersion = version;
+  }
+
   load_compiler(done) {
     if (compilerLoaded) {
       done();
@@ -16,7 +22,13 @@ class SolcW {
       compilerLoaded = true;
       done();
     });
-    solcProcess.send({action: 'loadCompiler'});
+    npm.getPackageVersion('solc', '0.4.10', false, function(location) {
+      console.log("new compiler installed at " + location);
+      //let requirePath = path.join(process.env.PWD, location.substr(2));
+      let requirePath = path.join(process.env.PWD, location);
+      console.log(requirePath);
+      solcProcess.send({action: 'loadCompiler', solcLocation: requirePath});
+    });
   }
 
   isCompilerLoaded() {

lib/core/config.js

@@ -1,4 +1,5 @@
 var fs = require('./fs.js');
+var File = require('./file.js');
 var Plugins = require('./plugins.js');
 var utils = require('../utils/utils.js');
 var npm = require('../pipeline/npm.js');
@@ -45,9 +46,9 @@ Config.prototype.loadConfigFiles = function(options) {
   this.loadStorageConfigFile();
   this.loadCommunicationConfigFile();
 
+  this.loadContractsConfigFile();
   this.loadPipelineConfigFile();
 
-  this.loadContractsConfigFile();
   this.loadWebServerConfigFile();
   this.loadChainTrackerFile();
   this.loadPluginContractFiles();
@@ -58,8 +59,8 @@ Config.prototype.reloadConfig = function() {
   this.loadBlockchainConfigFile();
   this.loadStorageConfigFile();
   this.loadCommunicationConfigFile();
-  this.loadPipelineConfigFile();
   this.loadContractsConfigFile();
+  this.loadPipelineConfigFile();
   this.loadChainTrackerFile();
 };
 
@@ -234,34 +235,38 @@ Config.prototype.loadFiles = function(files) {
     if (file === 'embark.js') {
 
       if (self.blockchainConfig.enabled || self.communicationConfig.provider === 'whisper' || self.communicationConfig.available_providers.indexOf('whisper') >= 0) {
-        readFiles.push({filename: 'web3.js',   content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")});
+        let web3Version = self.contractsConfig.versions["web3.js"];
+        if (web3Version) {
+        //if (false) {
+          //readFiles.push(new File({filename: 'web3-' + web3Version + '.js', type: 'custom', resolver: function(callback) {
+          readFiles.push(new File({filename: 'web3.js', type: 'custom', resolver: function(callback) {
+            npm.getPackageVersion('web3', web3Version, 'dist/web3.js', function(web3Content) {
+              callback(web3Content);
+            });
+          }}));
+        } else {
+          readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"}));
+        }
       }
 
       if (self.storageConfig.enabled && (self.storageConfig.provider === 'ipfs' || self.storageConfig.available_providers.indexOf('ipfs') >= 0)) {
-        readFiles.push({filename: 'ipfs.js',   content: fs.readFileSync(fs.embarkPath("js/ipfs.js")).toString(), path: fs.embarkPath("js/ipfs.js")});
+        readFiles.push(new File({filename: 'ipfs.js', type: 'embark_internal', path: "js/ipfs.js"}));
       }
 
       if (self.communicationConfig.enabled && (self.communicationConfig.provider === 'orbit' || self.communicationConfig.available_providers.indexOf('orbit') >= 0)) {
         // TODO: remove duplicated files if functionality is the same for storage and orbit
-        readFiles.push({filename: 'ipfs-api.js',   content: fs.readFileSync(fs.embarkPath("js/ipfs-api.min.js")).toString(), path: fs.embarkPath("js/ipfs-api.min.js")});
+        readFiles.push(new File({filename: 'ipfs-api.js', type: 'embark_internal', path: "js/ipfs-api.min.js"}));
-        readFiles.push({filename: 'orbit.js',  content: fs.readFileSync(fs.embarkPath("js/orbit.min.js")).toString(), path: fs.embarkPath("js/orbit.min.js")});
+        readFiles.push(new File({filename: 'orbit.js', type: 'embark_internal', path: "js/orbit.min.js"}));
       }
 
-      readFiles.push({filename: 'embark.js', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")});
+      readFiles.push(new File({filename: 'embark.js', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
     }
     if (file === '$EMBARK_JS') {
-      readFiles.push({filename: '$EMBARK_JS', content: fs.readFileSync(fs.embarkPath("js/build/embark.bundle.js")).toString(), path: fs.embarkPath("js/build/embark.bundle.js")});
+      readFiles.push(new File({filename: '$EMBARK_JS', type: 'embark_internal', path: "js/build/embark.bundle.js"}));
-    }
-    if (file.indexOf("web3-") === 0) {
-        let web3Version = file.split('web3-')[1].split(".js")[0];
-        npm.getPackageVersion('web3', web3Version, function(web3Content) {
-          self.logger.error('downloaded web3');
-          readFiles.push({filename: file, content: web3Content, path: fs.embarkPath("js/web3.js")});
-        });
-    }
-    if (file === "web3.js") {
-      readFiles.push({filename: 'web3.js',   content: fs.readFileSync(fs.embarkPath("js/web3.js")).toString(), path: fs.embarkPath("js/web3.js")});
     }
+    //if (file === "web3.js") {
+    //  readFiles.push(new File({filename: 'web3.js', type: 'embark_internal', path: "js/web3.js"));
+    //}
   });
 
   // get plugins
@@ -298,13 +303,13 @@ Config.prototype.loadFiles = function(files) {
       return;
     } else if (file.indexOf("web3") === 0) {
       return;
-    } else if (file === 'abi.js') {
+    //} else if (file === 'abi.js') {
-      readFiles.push({filename: file, content: "", path: file});
+    //  readFiles.push({filename: file, content: "", path: file});
     } else {
       if (file.indexOf('.') >= 0)  {
-        readFiles.push({filename: file, content: fs.readFileSync(file).toString(), path: file});
+        readFiles.push(new File({filename: file, type: "dapp_file", path: file}));
       } else if (file[0] === '$') {
-        readFiles.push({filename: file, content: "", path: file});
+        //readFiles.push(new File({filename: file, type: 'embark_internal', path: file}));
       }
     }
   });
@@ -320,7 +325,10 @@ Config.prototype.loadPluginContractFiles = function() {
     contractsPlugins.forEach(function(plugin) {
       plugin.contractsFiles.forEach(function(file) {
         var filename = file.replace('./','');
-        self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)});
+        //self.contractsFiles.push({filename: filename, content: plugin.loadPluginFile(file), path: plugin.pathToFile(file)});
+        self.contractsFiles.push(new File({filename: filename, type: 'custom', resolver: function(callback) {
+          callback(plugin.loadPluginFile(file));
+        }}));
       });
     });
   }

lib/core/engine.js

@@ -83,9 +83,10 @@ class Engine {
     this.events.on('abi', function (abi, contractsJSON) {
       self.currentAbi = abi;
       self.contractsJSON = contractsJSON;
-      pipeline.build(abi, contractsJSON);
+      pipeline.build(abi, contractsJSON, null, function() {
         self.events.emit('outputDone');
       });
+    });
     // TODO: still need to redeploy contracts because the original contracts
     // config is being corrupted
     //this.events.on('file-event', function(fileType, path) {

lib/core/file.js

@@ -0,0 +1,26 @@
+let fs = require('./fs.js');
+
+class File {
+
+  constructor(options) {
+    this.filename = options.filename;
+    this.type = options.type;
+    this.path = options.path;
+    this.resolver = options.resolver;
+  }
+
+  content(callback) {
+    if (this.type === 'embark_internal') {
+      return callback(fs.readFileSync(fs.embarkPath(this.path)).toString());
+    } else if (this.type === 'dapp_file') {
+      return callback(fs.readFileSync(this.path).toString());
+    } else if (this.type === 'custom') {
+      return this.resolver(callback);
+    } else {
+      throw new Error("unknown file: " + this.filename);
+    }
+  }
+
+}
+
+module.exports = File;

lib/core/fs.js

@@ -34,6 +34,10 @@ function embarkPath(fileOrDir) {
   return utils.joinPath(__dirname, '/../../', fileOrDir);
 }
 
+function dappPath() {
+  return process.env.PWD;
+}
+
 module.exports = {
   mkdirpSync: mkdirpSync,
   copySync: copySync,

lib/pipeline/npm.js

@@ -1,12 +1,16 @@
 let utils = require('../utils/utils.js');
+let fs = require('../core/fs.js');
+let o_fs = require('fs-extra');
+
+let http = require('follow-redirects').http;
+let https = require('follow-redirects').https;
+var tar = require('tar');
 
 module.exports = {
-  // TODO: need to refactor to make it truly generalistic, perhaps by
+  getPackageVersion: function(packageName, version, returnContent, callback) {
-  // downloading the tarball itself
+    let npmRegistry = "https://registry.npmjs.org/" + packageName + "/" + version;
-  getPackageVersion: function(packageName, version, callback) {
-    let npmRegistry = "http://registry.npmjs.org/" + packageName + "/" + version;
 
-    utils.httpGet(npmRegistry, function (res) {
+    utils.httpsGet(npmRegistry, function (res) {
 
       let body = '';
       res.on('data', function (d) {
@@ -14,19 +18,53 @@ module.exports = {
       });
       res.on('end', function () {
         let registryJSON = JSON.parse(body);
-        let gitHash = registryJSON.gitHead;
-        let repo = registryJSON.homepage.split("github.com/")[1];
 
-        let gitUrl = "http://raw.githubusercontent.com/" + repo + "/" + gitHash + "/dist/" + packageName + ".js";
+        let tarball = registryJSON.dist.tarball;
-        utils.httpGet(gitUrl, function (res2) {
+
-          let body = '';
+        var download = function(url, dest, cb) {
-          res2.on('data', function (d) {
+          var file = o_fs.createWriteStream(dest);
-            body += d;
+          var request = (url.substring(0,5) === 'https' ? https : http).get(url, function(response) {
+            response.pipe(file);
+            file.on('finish', function() {
+              file.close(cb);  // close() is async, call cb after close completes.
             });
-          res2.on('end', function () {
+          }).on('error', function(err) { // Handle errors
-            callback(body);
+            fs.unlink(dest); // Delete the file async. (But we don't check the result)
+            if (cb) cb(err.message);
           });
+        };
+
+        let packageDirectory = './.embark/versions/' + packageName + '/' + version + '/';
+
+        if (fs.existsSync(packageDirectory + "/downloaded_package.tgz")) {
+          if (returnContent) {
+            let distFile = packageDirectory + returnContent;
+            callback(fs.readFileSync(distFile).toString());
+          } else {
+            callback(packageDirectory);
+          }
+        } else {
+          fs.mkdirpSync(packageDirectory);
+          //self.logger.info("downloading " + packageName + " " + version + "....");
+
+          download(tarball, packageDirectory + "/downloaded_package.tgz", function() {
+            o_fs.createReadStream(packageDirectory + '/downloaded_package.tgz').pipe(
+              tar.x({
+              strip: 1,
+              C: packageDirectory
+            }).on('end', function() {
+              if (returnContent) {
+                let distFile = packageDirectory + returnContent;
+                callback(fs.readFileSync(distFile).toString());
+              } else {
+                callback(packageDirectory);
+              }
+            })
+            );
           });
+
+        }
+
       });
     });
   }

lib/pipeline/pipeline.js

@@ -1,5 +1,6 @@
 /*jshint esversion: 6, loopfunc: true */
 let fs = require('../core/fs.js');
+let async = require('async');
 
 class Pipeline {
 
@@ -11,56 +12,71 @@ class Pipeline {
     this.plugins = options.plugins;
   }
 
-  build(abi, contractsJSON, path) {
+  build(abi, contractsJSON, path, callback) {
     let self = this;
 
     this.buildContracts(contractsJSON);
 
-    for (let targetFile in this.assetFiles) {
+    // limit:1 due to issues when downloading required files such as web3.js
+    async.eachOfLimit(this.assetFiles, 1, function (files, targetFile, cb) {
 
-      let contentFiles = this.assetFiles[targetFile].map(file => {
+      // limit:1 due to issues when downloading required files such as web3.js
+      async.mapLimit(files, 1,
+        function(file, fileCb) {
           self.logger.trace("reading " + file.filename);
 
-        let pipelinePlugins = this.plugins.getPluginsFor('pipeline');
+          let pipelinePlugins = self.plugins.getPluginsFor('pipeline');
 
           if (file.filename === "$ALL_CONTRACTS") {
-          return {content: abi, filename: file.filename, path: file.path, modified: true};
+            return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
           } else if (file.filename === "$EMBARK_JS") {
-          return {content: file.content, filename: "embark.js", path: file.path, modified: true};
+            return file.content(function(fileContent) {
+              return fileCb(null, {content: fileContent, filename: "embark.js", path: file.path, modified: true});
+            });
           } else if (file.filename[0] === '$') {
             let contractName = file.filename.substr(1);
-          return {content: this.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true};
+            return fileCb(null, {content: self.buildContractJS(contractName), filename: contractName + ".js", path: file.path, modified: true});
           } else if (file.filename === 'embark.js') {
-          return {content: file.content + "\n" + abi, filename: file.filename, path: file.path, modified: true};
+            return file.content(function(fileContent) {
+              return fileCb(null, {content: fileContent + "\n" + abi, filename: file.filename, path: file.path, modified: true});
+            });
           } else if (file.filename === 'abi.js') {
-          return {content: abi, filename: file.filename, path: file.path, modified: true};
+            return fileCb(null, {content: abi, filename: file.filename, path: file.path, modified: true});
           } else if (['web3.js', 'ipfs.js', 'ipfs-api.js', 'orbit.js'].indexOf(file.filename) >= 0) {
-          file.modified = true;
+            file.content(function(fileContent) {
-          return file;
+              return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
+            });
           } else {
 
             if (pipelinePlugins.length > 0) {
-            pipelinePlugins.forEach(function (plugin) {
+              file.content(function(fileContent) {
-              try {
+                async.eachSeries(pipelinePlugins, function(plugin, pluginCB) {
                   if (file.options && file.options.skipPipeline) {
-                  return;
+                    return pluginCB();
                   }
-                file.content = plugin.runPipeline({targetFile: file.filename, source: file.content});
+
+                  fileContent = plugin.runPipeline({targetFile: file.filename, source: fileContent});
                   file.modified = true;
-              }
+                  pluginCB();
-              catch (err) {
+                },
+                function (err) {
+                  if (err) {
                     self.logger.error(err.message);
                   }
+                  return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
+                });
+              });
+            } else {
+              file.content(function(fileContent) {
+                return fileCb(null, {content: fileContent, filename: file.filename, path: file.path, modified: true});
               });
             }
-
-          return file;
           }
-      });
+        },
-
+        function (err, contentFiles) {
           let dir = targetFile.split('/').slice(0, -1).join('/');
-      self.logger.trace("creating dir " + this.buildDir + dir);
+          self.logger.trace("creating dir " + self.buildDir + dir);
-      fs.mkdirpSync(this.buildDir + dir);
+          fs.mkdirpSync(self.buildDir + dir);
 
           // if it's a directory
           if (targetFile.slice(-1) === '/' || targetFile.indexOf('.') === -1) {
@@ -79,14 +95,22 @@ class Pipeline {
             });
           } else {
             let content = contentFiles.map(function (file) {
+              if (file === undefined) {
+                return "";
+              }
               return file.content;
             }).join("\n");
 
-        self.logger.info("writing file " + (this.buildDir + targetFile).bold.dim);
+            self.logger.info("writing file " + (self.buildDir + targetFile).bold.dim);
-        fs.writeFileSync(this.buildDir + targetFile, content);
+            fs.writeFileSync(self.buildDir + targetFile, content);
           }
+          cb();
         }
-
+      );
+    },
+    function (err, results) {
+      callback();
+    });
   }
 
   buildContracts(contractsJSON) {

lib/utils/utils.js

@@ -3,6 +3,7 @@ let path = require('path');
 let globule = require('globule');
 let merge = require('merge');
 let http = require('follow-redirects').http;
+let https = require('follow-redirects').https;
 let shelljs = require('shelljs');
 
 function joinPath() {
@@ -33,6 +34,10 @@ function httpGet(url, callback) {
   return http.get(url, callback);
 }
 
+function httpsGet(url, callback) {
+  return https.get(url, callback);
+}
+
 function runCmd(cmd, options) {
   let result = shelljs.exec(cmd, options || {silent: true});
   if (result.code !== 0) {
@@ -65,6 +70,7 @@ module.exports = {
   recursiveMerge: recursiveMerge,
   checkIsAvailable: checkIsAvailable,
   httpGet: httpGet,
+  httpsGet: httpsGet,
   runCmd: runCmd,
   cd: cd,
   sed: sed,

package.json

@@ -33,6 +33,7 @@
     "serve-static": "^1.11.1",
     "shelljs": "^0.5.0",
     "solc": "0.4.11",
+    "tar": "^3.1.5",
     "toposort": "^1.0.0",
     "underscore": "^1.8.3",
     "underscore.string": "^3.3.4",

test_app/config/contracts.json

@@ -1,5 +1,17 @@
 {
   "default": {
+    "versions": {
+      "web3.js": "0.19.1",
+      "solc": "0.4.11"
+    },
+    "deployment": {
+      "host": "localhost",
+      "port": 8545
+    },
+    "dappConnection": [
+      "$WEB3",
+      "localhost:8545"
+    ],
     "gas": "auto",
     "contracts": {
       "SimpleStorage": {

test_app/package.json

@@ -9,8 +9,7 @@
   "author": "",
   "license": "ISC",
   "homepage": "",
-  "devDependencies": {
+  "devDependencies": {},
-  },
   "dependencies": {
     "embark-babel": "^1.0.0",
     "embark-service": "./extensions/embark-service",