download files in full path of url

lib/core/config.js

@@ -152,7 +152,7 @@ Config.prototype.getExternalContractUrl = function (contract) {
     const match = contract.file.match(/https:\/\/github\.[a-z]+\/(.*)/);
     if (!match) {
       this.logger.error(MALFORMED_ERROR + contract.file);
-      return '';
+      return null;
     }
     url = `${RAW_URL}${match[1].replace('blob/', '')}`;
   } else if (contract.file.startsWith('git')) {
@@ -164,11 +164,11 @@ Config.prototype.getExternalContractUrl = function (contract) {
     // [4] path
     // [5] branch
     const match = contract.file.match(
-      /(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-7]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
+      /(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
     );
     if (!match) {
       this.logger.error(MALFORMED_ERROR + contract.file);
-      return '';
+      return null;
     }
     let branch = match[5];
     if (!branch) {
@@ -178,7 +178,13 @@ Config.prototype.getExternalContractUrl = function (contract) {
   } else {
     url = contract.file;
   }
-  return url;
+  const match = url.match(
+    /\.[a-z]+\/([a-zA-Z0-9_\-\/.]+)/
+  );
+  return {
+    url,
+    filePath: match[1]
+  };
 };
 
 Config.prototype.loadExternalContractsFiles = function() {
@@ -189,9 +195,12 @@ Config.prototype.loadExternalContractsFiles = function() {
       continue;
     }
     if (contract.file.startsWith('http') || contract.file.startsWith('git')) {
-      const url = this.getExternalContractUrl(contract);
+      const fileObj = this.getExternalContractUrl(contract);
-      const localFile = httpContractDir + path.basename(url);
+      if (!fileObj) {
-      this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: url}));
+        return this.logger.error("HTTP contract file not found: " + contract.file);
+      }
+      const localFile = httpContractDir + fileObj.filePath;
+      this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: fileObj.url}));
     } else if (fs.existsSync(contract.file)) {
       this.contractsFiles.push(new File({filename: contract.file, type: File.types.dapp_file, basedir: '', path: contract.file}));
     } else if (fs.existsSync(path.join('./node_modules/', contract.file))) {

lib/core/file.js

@@ -20,16 +20,23 @@ class File {
     }
     const regex = /import "([a-zA-Z0-9_\-.\\\/]+)";/g;
     let matches;
+    const filesToDownload = [];
+    const pathWithoutFile = path.dirname(this.path);
     while ((matches = regex.exec(content))) {
       console.log('Need to download', matches[1]);
+      filesToDownload.push({
+        fileRelativePath: matches[1],
+        url: path.join(pathWithoutFile, matches[1])
+      });
     }
+    //async.each(filesToDownload, (file))
   }
 
-  downloadFile (filename, callback) {
+  downloadFile (filename, url, callback) {
-    const self = this;
+    // const self = this;
     async.waterfall([
       function makeTheDir(next) {
-        fs.mkdirp(path.dirname(self.filename), (err) => {
+        fs.mkdirp(path.dirname(filename), (err) => {
           if (err) {
             return next(err);
           }
@@ -37,28 +44,24 @@ class File {
         });
       },
       function downloadTheFile(next) {
-        request(self.path)
+        request(url)
           .on('response', function (response) {
             if (response.statusCode !== 200) {
               next('Getting file returned code ' + response.statusCode);
             }
           })
           .on('error', next)
-          .pipe(fs.createWriteStream(self.filename))
+          .pipe(fs.createWriteStream(filename))
-          .on('finish', () => {
+          .on('finish', next);
-            self.path = self.filename;
-            self.type = File.types.dapp_file;
-            next();
-          });
       },
       function readFile(next) {
-        fs.readFile(self.path, next);
+        fs.readFile(filename, next);
-      },
+      }// ,
-      function parseForImports(content, next) {
+      /*function parseForImports(content, next) {
         self.parseFileForImport(content, (err) => {
           next(err, content);
         });
-      }
+      }*/
     ], (err, content) => {
       if (err) {
         console.error('Error while downloading the file', err);
@@ -76,7 +79,14 @@ class File {
     } else if (this.type === File.types.custom) {
       return this.resolver(callback);
     } else if (this.type === File.types.http) {
-      this.downloadFile(callback);
+      this.downloadFile(this.filename, this.path, (content) => {
+        if (!content) {
+          return callback(content);
+        }
+        this.path = this.filename;
+        this.type = File.types.dapp_file;
+        callback(content);
+      });
     } else {
       throw new Error("unknown file: " + this.filename);
     }

test/config.js

@@ -59,72 +59,89 @@ describe('embark.Config', function () {
 
   describe('#getExternalContractUrl', function () {
     it('should get the right url for a https://github file', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'https://github.com/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
       );
-      assert.strictEqual(url,
+      assert.deepEqual(fileObj,
-        'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol');
+        {
+          filePath: 'embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
+          url: 'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol'
+        });
     });
 
     it('should fail for a malformed https://github file', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'https://github/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
       );
-      assert.strictEqual(url, '');
+      assert.strictEqual(fileObj, null);
     });
 
     it('should get the right url for a git:// file with no branch #', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol'}
       );
-      console.log(url);
+      assert.deepEqual(fileObj,
-      assert.strictEqual(url,
+        {
-        'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol');
+          filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
+        });
     });
 
     it('should get the right url for a git:// file with a branch #', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch'}
       );
-      assert.strictEqual(url,
+      assert.deepEqual(fileObj,
-        'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol');
+        {
+          filePath: 'status-im/contracts/myBranch/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol'
+        });
     });
 
     it('should fail when the git:// file is malformed', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'git://github.com/identity/ERC725.sol#myBranch'}
       );
-      assert.strictEqual(url, '');
+      assert.strictEqual(fileObj, null);
     });
 
     it('should get the right url with a github.com file without branch #', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol'}
       );
-      assert.strictEqual(url,
+      assert.deepEqual(fileObj,
-        'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol');
+        {
+          filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
+        });
     });
 
     it('should get the right url with a github.com file with branch #', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
       );
-      assert.strictEqual(url,
+      assert.deepEqual(fileObj,
-        'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol');
+        {
+          filePath: 'status-im/contracts/theBranch/contracts/identity/ERC725.sol',
+          url: 'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol'
+        });
     });
 
     it('should fail with a malformed github.com url', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
       );
-      assert.strictEqual(url, '');
+      assert.strictEqual(fileObj, null);
     });
 
     it('should succeed with a generic http url', function () {
-      const url = config.getExternalContractUrl(
+      const fileObj = config.getExternalContractUrl(
         {file: 'http://myurl.com/myFile.sol'}
       );
-      assert.strictEqual(url, 'http://myurl.com/myFile.sol');
+      assert.deepEqual(fileObj, {
+        filePath: 'myFile.sol',
+        url: 'http://myurl.com/myFile.sol'
+      });
     });
   });
 

test/contracts/simple_storage.sol

@@ -1,8 +1,8 @@
 pragma solidity ^0.4.7;
 contract SimpleStorage {
   uint public storedData;
-  import "ownable.sol";
+  import "./ownable.sol";
-  import "ownable2.sol";
+  import "../../contracts/token.sol";
 
   function SimpleStorage(uint initialValue) {
     storedData = initialValue;

test/file.js

@@ -6,7 +6,8 @@ describe('embark.File', function () {
   describe('parseFileForImport', () => {
     it('should find all the imports', function () {
       const contract = fs.readFileSync('./test/contracts/simple_storage.sol').toString();
-      const file = new File({filename: 'simple_storage.sol'});
+      const file = new File({filename: 'simple_storage.sol',
+        path: 'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/test_app/app/contracts/simple_storage.sol'});
       file.parseFileForImport(contract);
     });
   });