add code to parse all files to check for http imports

lib/core/config.js

@@ -143,49 +143,6 @@ Config.prototype.loadContractsConfigFile = function() {
   this.contractsConfig = this._mergeConfig(configFilePath, configObject, this.env);
 };
 
-Config.prototype.getExternalContractUrl = function (contract) {
-  let url;
-  const RAW_URL = 'https://raw.githubusercontent.com/';
-  const MALFORMED_ERROR = 'Malformed Github URL for ';
-  if (contract.file.startsWith('https://github')) {
-    const match = contract.file.match(/https:\/\/github\.[a-z]+\/(.*)/);
-    if (!match) {
-      this.logger.error(MALFORMED_ERROR + contract.file);
-      return null;
-    }
-    url = `${RAW_URL}${match[1].replace('blob/', '')}`;
-  } else if (contract.file.startsWith('git')) {
-    // Match values
-    // [0] entire input
-    // [1] git://
-    // [2] user
-    // [3] repository
-    // [4] path
-    // [5] branch
-    const match = contract.file.match(
-      /(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
-    );
-    if (!match) {
-      this.logger.error(MALFORMED_ERROR + contract.file);
-      return null;
-    }
-    let branch = match[5];
-    if (!branch) {
-      branch = 'master';
-    }
-    url = `${RAW_URL}${match[2]}/${match[3]}/${branch}/${match[4]}`;
-  } else {
-    url = contract.file;
-  }
-  const match = url.match(
-    /\.[a-z]+\/([a-zA-Z0-9_\-\/.]+)/
-  );
-  return {
-    url,
-    filePath: match[1]
-  };
-};
-
 Config.prototype.loadExternalContractsFiles = function() {
   let contracts = this.contractsConfig.contracts;
   for (let contractName in contracts) {
@@ -194,11 +151,11 @@ Config.prototype.loadExternalContractsFiles = function() {
       continue;
     }
     if (contract.file.startsWith('http') || contract.file.startsWith('git')) {
-      const fileObj = this.getExternalContractUrl(contract);
+      const fileObj = utils.getExternalContractUrl(contract.file);
       if (!fileObj) {
         return this.logger.error("HTTP contract file not found: " + contract.file);
       }
-      const localFile = constants.httpContractsDirectory + fileObj.filePath;
+      const localFile = fileObj.filePath;
       this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: fileObj.url}));
     } else if (fs.existsSync(contract.file)) {
       this.contractsFiles.push(new File({filename: contract.file, type: File.types.dapp_file, basedir: '', path: contract.file}));

lib/core/file.js

@@ -2,6 +2,7 @@ const async = require('async');
 const fs = require('./fs.js');
 const path = require('path');
 const request = require('request');
+const utils = require('../utils/utils');
 
 class File {
 
@@ -13,21 +14,34 @@ class File {
     this.resolver = options.resolver;
   }
 
-  parseFileForImport(content, callback) {
+  parseFileForImport(content, isHttpContract, callback) {
+    if (typeof isHttpContract === 'function') {
+      callback = isHttpContract;
+      isHttpContract = false;
+    }
     const self = this;
     if (self.filename.indexOf('.sol') < 0) {
       // Only supported in Solidity
       return callback();
     }
-    const regex = /import "([a-zA-Z0-9_\-.\\\/]+)";/g;
+    const regex = /import "([a-zA-Z0-9_\-.\\\/:]+)";/g;
     let matches;
     const filesToDownload = [];
     const pathWithoutFile = path.dirname(self.path);
     while ((matches = regex.exec(content))) {
-      filesToDownload.push({
+      const httpFileObj = utils.getExternalContractUrl(matches[1]);
+      const fileObj = {
         fileRelativePath: path.join(path.dirname(self.filename), matches[1]),
         url: `${pathWithoutFile}/${matches[1]}`
-      });
+      };
+      if (httpFileObj) {
+        fileObj.fileRelativePath = httpFileObj.filePath;
+        fileObj.url = httpFileObj.url;
+      } else if (!isHttpContract) {
+        // Just a normal import
+        continue;
+      }
+      filesToDownload.push(fileObj);
     }
 
     async.each(filesToDownload, ((fileObj, eachCb) => {
@@ -63,7 +77,7 @@ class File {
         fs.readFile(filename, next);
       },
       function parseForImports(content, next) {
-        self.parseFileForImport(content, (err) => {
+        self.parseFileForImport(content, true, (err) => {
           next(err, content);
         });
       }
@@ -77,17 +91,27 @@ class File {
   }
 
   content (callback) {
+    let content;
     if (this.type === File.types.embark_internal) {
-      return callback(fs.readFileSync(fs.embarkPath(this.path)).toString());
+      content = fs.readFileSync(fs.embarkPath(this.path)).toString();
     } else if (this.type === File.types.dapp_file) {
-      return callback(fs.readFileSync(this.path).toString());
+      content = fs.readFileSync(this.path).toString();
     } else if (this.type === File.types.custom) {
-      return this.resolver(callback);
+      return this.resolver((theContent) => {
+        if (!this.parsedImports) {
+          this.parsedImports = true;
+          return this.parseFileForImport(content, () => {
+            callback(theContent);
+          });
+        }
+        callback(theContent);
+      });
     } else if (this.type === File.types.http) {
-      this.downloadFile(this.filename, this.path, (content) => {
+      return this.downloadFile(this.filename, this.path, (content) => {
         if (!content) {
           return callback(content);
         }
+        this.parsedImports = true;
         this.path = this.filename;
         this.type = File.types.dapp_file;
         callback(content);
@@ -95,6 +119,13 @@ class File {
     } else {
       throw new Error("unknown file: " + this.filename);
     }
+    if (!this.parsedImports) {
+      this.parsedImports = true;
+      return this.parseFileForImport(content, () => {
+        callback(content);
+      });
+    }
+    callback(content);
   }
 
 }

lib/utils/utils.js

@@ -6,6 +6,7 @@ let https = require('follow-redirects').https;
 let shelljs = require('shelljs');
 var tar = require('tar');
 var propose = require('propose');
+const constants = require('../constants');
 
 //let fs = require('../core/fs.js');
 let o_fs = require('fs-extra');
@@ -127,6 +128,51 @@ function pwd() {
   return process.env.PWD || process.cwd();
 }
 
+function getExternalContractUrl(file) {
+  let url;
+  const RAW_URL = 'https://raw.githubusercontent.com/';
+  const MALFORMED_ERROR = 'Malformed Github URL for ';
+  if (file.startsWith('https://github')) {
+    const match = file.match(/https:\/\/github\.[a-z]+\/(.*)/);
+    if (!match) {
+      console.error(MALFORMED_ERROR + file);
+      return null;
+    }
+    url = `${RAW_URL}${match[1].replace('blob/', '')}`;
+  } else if (file.startsWith('git')) {
+    // Match values
+    // [0] entire input
+    // [1] git://
+    // [2] user
+    // [3] repository
+    // [4] path
+    // [5] branch
+    const match = file.match(
+      /(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
+    );
+    if (!match) {
+      console.error(MALFORMED_ERROR + file);
+      return null;
+    }
+    let branch = match[5];
+    if (!branch) {
+      branch = 'master';
+    }
+    url = `${RAW_URL}${match[2]}/${match[3]}/${branch}/${match[4]}`;
+  } else if (file.startsWith('http')) {
+    url = file;
+  } else {
+    return null;
+  }
+  const match = url.match(
+    /\.[a-z]+\/([a-zA-Z0-9_\-\/.]+)/
+  );
+  return {
+    url,
+    filePath: constants.httpContractsDirectory + match[1]
+  };
+}
+
 module.exports = {
   joinPath: joinPath,
   filesMatchingPattern: filesMatchingPattern,
@@ -143,5 +189,6 @@ module.exports = {
   downloadFile: downloadFile,
   extractTar: extractTar,
   proposeAlternative: proposeAlternative,
-  pwd: pwd
+  pwd: pwd,
+  getExternalContractUrl
 };

test/config.js

@@ -57,94 +57,6 @@ describe('embark.Config', function () {
     });
   });
 
-  describe('#getExternalContractUrl', function () {
-    it('should get the right url for a https://github file', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'https://github.com/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
-      );
-      assert.deepEqual(fileObj,
-        {
-          filePath: 'embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
-          url: 'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol'
-        });
-    });
-
-    it('should fail for a malformed https://github file', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'https://github/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
-      );
-      assert.strictEqual(fileObj, null);
-    });
-
-    it('should get the right url for a git:// file with no branch #', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol'}
-      );
-      assert.deepEqual(fileObj,
-        {
-          filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
-          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
-        });
-    });
-
-    it('should get the right url for a git:// file with a branch #', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch'}
-      );
-      assert.deepEqual(fileObj,
-        {
-          filePath: 'status-im/contracts/myBranch/contracts/identity/ERC725.sol',
-          url: 'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol'
-        });
-    });
-
-    it('should fail when the git:// file is malformed', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'git://github.com/identity/ERC725.sol#myBranch'}
-      );
-      assert.strictEqual(fileObj, null);
-    });
-
-    it('should get the right url with a github.com file without branch #', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol'}
-      );
-      assert.deepEqual(fileObj,
-        {
-          filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
-          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
-        });
-    });
-
-    it('should get the right url with a github.com file with branch #', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
-      );
-      assert.deepEqual(fileObj,
-        {
-          filePath: 'status-im/contracts/theBranch/contracts/identity/ERC725.sol',
-          url: 'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol'
-        });
-    });
-
-    it('should fail with a malformed github.com url', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
-      );
-      assert.strictEqual(fileObj, null);
-    });
-
-    it('should succeed with a generic http url', function () {
-      const fileObj = config.getExternalContractUrl(
-        {file: 'http://myurl.com/myFile.sol'}
-      );
-      assert.deepEqual(fileObj, {
-        filePath: 'myFile.sol',
-        url: 'http://myurl.com/myFile.sol'
-      });
-    });
-  });
-
   describe('#loadExternalContractsFiles', function () {
     it('should create the right list of files and download', function () {
       config.contractsFiles = [];

test/contracts/contract_with_http_import.sol

@@ -0,0 +1,19 @@
+pragma solidity ^0.4.7;
+contract SimpleStorage {
+  uint public storedData;
+  import "https://github.com/embark-framework/embark/blob/develop/test_apps/contracts_app/contracts/contract_args.sol";
+
+  function SimpleStorage(uint initialValue) {
+    storedData = initialValue;
+  }
+
+  function set(uint x) {
+    storedData = x;
+  }
+
+  function get() constant returns (uint retVal) {
+    return storedData;
+  }
+
+}
+

test/file.js

@@ -16,7 +16,7 @@ describe('embark.File', function () {
           cb();
         });
 
-      file.parseFileForImport(contract, () => {
+      file.parseFileForImport(contract, true, () => {
         assert.strictEqual(downloadFileStub.callCount, 1);
         assert.strictEqual(downloadFileStub.firstCall.args[0],
           path.normalize('.embark/contracts/embark-framework/embark/master/test_app/app/contracts/ownable.sol'));
@@ -25,5 +25,39 @@ describe('embark.File', function () {
         done();
       });
     });
+
+    it('should find all the imports but not call download because not a http contract', function (done) {
+      const contract = fs.readFileSync('./test/contracts/contract_with_import.sol').toString();
+      const file = new File({filename: '.embark/contracts/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
+        path: 'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/test_app/app/contracts/simple_storage.sol'});
+      const downloadFileStub = sinon.stub(file, 'downloadFile')
+        .callsFake((path, url, cb) => {
+          cb();
+        });
+
+      file.parseFileForImport(contract, () => {
+        assert.strictEqual(downloadFileStub.callCount, 0);
+        done();
+      });
+    });
+
+    it('should find all the imports and call downlaod because it is an http import', function (done) {
+      const contract = fs.readFileSync('./test/contracts/contract_with_http_import.sol').toString();
+      const file = new File({filename: '.embark/contracts/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
+        path: 'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/test_app/app/contracts/simple_storage.sol'});
+      const downloadFileStub = sinon.stub(file, 'downloadFile')
+        .callsFake((path, url, cb) => {
+          cb();
+        });
+
+      file.parseFileForImport(contract, () => {
+        assert.strictEqual(downloadFileStub.callCount, 1);
+        assert.strictEqual(downloadFileStub.firstCall.args[0],
+          '.embark/contracts/embark-framework/embark/develop/test_apps/contracts_app/contracts/contract_args.sol');
+        assert.strictEqual(downloadFileStub.firstCall.args[1],
+          'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/contracts_app/contracts/contract_args.sol');
+        done();
+      });
+    });
   });
 });

test/test.js

@@ -0,0 +1,94 @@
+/*global describe, it*/
+const Utils = require('../lib/utils/utils');
+const assert = require('assert');
+const constants = require('../lib/constants');
+
+describe('embark.utils', function () {
+  describe('#getExternalContractUrl', function () {
+    it('should get the right url for a https://github file', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'https://github.com/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'
+      );
+      assert.deepEqual(fileObj,
+        {
+          filePath: constants.httpContractsDirectory + 'embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
+          url: 'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol'
+        });
+    });
+
+    it('should fail for a malformed https://github file', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'https://github/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'
+      );
+      assert.strictEqual(fileObj, null);
+    });
+
+    it('should get the right url for a git:// file with no branch #', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'git://github.com/status-im/contracts/contracts/identity/ERC725.sol'
+      );
+      assert.deepEqual(fileObj,
+        {
+          filePath: constants.httpContractsDirectory + 'status-im/contracts/master/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
+        });
+    });
+
+    it('should get the right url for a git:// file with a branch #', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch'
+      );
+      assert.deepEqual(fileObj,
+        {
+          filePath: constants.httpContractsDirectory + 'status-im/contracts/myBranch/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol'
+        });
+    });
+
+    it('should fail when the git:// file is malformed', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'git://github.com/identity/ERC725.sol#myBranch'
+      );
+      assert.strictEqual(fileObj, null);
+    });
+
+    it('should get the right url with a github.com file without branch #', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'github.com/status-im/contracts/contracts/identity/ERC725.sol'
+      );
+      assert.deepEqual(fileObj,
+        {
+          filePath: constants.httpContractsDirectory + 'status-im/contracts/master/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
+        });
+    });
+
+    it('should get the right url with a github.com file with branch #', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch'
+      );
+      assert.deepEqual(fileObj,
+        {
+          filePath: constants.httpContractsDirectory + 'status-im/contracts/theBranch/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol'
+        });
+    });
+
+    it('should fail with a malformed github.com url', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch'
+      );
+      assert.strictEqual(fileObj, null);
+    });
+
+    it('should succeed with a generic http url', function () {
+      const fileObj = Utils.getExternalContractUrl(
+        'http://myurl.com/myFile.sol'
+      );
+      assert.deepEqual(fileObj, {
+        filePath: constants.httpContractsDirectory + 'myFile.sol',
+        url: 'http://myurl.com/myFile.sol'
+      });
+    });
+  });
+});