include .sol import from ipfs

lib/core/config.js

@@ -31,12 +31,13 @@ var Config = function(options) {
   this.context = options.context || [constants.contexts.any];
   this.shownNoAccountConfigMsg = false; // flag to ensure "no account config" message is only displayed once to the user
   this.corsParts = [];
-
+  this.ipfsUrl = null;
   this.events.setCommandHandler("config:cors:add", (url) => {
     this.corsParts.push(url);
     this._updateBlockchainCors();
   });
 
+
   self.events.setCommandHandler("config:contractsConfig", (cb) => {
     cb(self.contractsConfig);
   });
@@ -79,6 +80,7 @@ Config.prototype.loadConfigFiles = function(options) {
   this.loadEmbarkConfigFile();
   this.loadBlockchainConfigFile();
   this.loadStorageConfigFile();
+  this.loadContractFiles();
   this.loadCommunicationConfigFile();
   this.loadNameSystemConfigFile();
   this.loadPipelineConfigFile();
@@ -96,6 +98,7 @@ Config.prototype.reloadConfig = function() {
   this.loadEmbarkConfigFile();
   this.loadBlockchainConfigFile();
   this.loadStorageConfigFile();
+  this.loadContractFiles();
   this.loadCommunicationConfigFile();
   this.loadNameSystemConfigFile();
   this.loadPipelineConfigFile();
@@ -107,6 +110,18 @@ Config.prototype.reloadConfig = function() {
   this._updateBlockchainCors();
 };
 
+Config.prototype.loadContractFiles = function() {
+  const contracts = this.embarkConfig.contracts;
+  const newContractsFiles = this.loadFiles(contracts);
+  if (!this.contractFiles || newContractsFiles.length !== this.contractFiles.length || !deepEqual(newContractsFiles, this.contractFiles)) {
+    this.contractsFiles = this.contractsFiles.concat(newContractsFiles).filter((file, index, arr) => {
+      return !arr.some((file2, index2) => {
+        return file.filename === file2.filename && index < index2;
+      });
+    });
+  }
+};
+
 Config.prototype._updateBlockchainCors = function(){
   let blockchainConfig = this.blockchainConfig;
   let storageConfig = this.storageConfig;
@@ -303,22 +318,26 @@ Config.prototype.loadContractsConfigFile = function() {
 
 Config.prototype.loadExternalContractsFiles = function() {
   let contracts = this.contractsConfig.contracts;
+  let storageConfig = this.storageConfig;
+  if (storageConfig && storageConfig.upload && storageConfig.upload.getUrl) {
+      this.ipfsUrl = storageConfig.upload.getUrl;
+  }
   for (let contractName in contracts) {
     let contract = contracts[contractName];
     if (!contract.file) {
       continue;
     }
-    if (contract.file.startsWith('http') || contract.file.startsWith('git')) {
+    if (contract.file.startsWith('http') || contract.file.startsWith('git') || contract.file.startsWith('ipfs')) {
-      const fileObj = utils.getExternalContractUrl(contract.file);
+      const fileObj = utils.getExternalContractUrl(contract.file,this.ipfsUrl);
       if (!fileObj) {
         return this.logger.error(__("HTTP contract file not found") + ": " + contract.file);
       }
       const localFile = fileObj.filePath;
-      this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: fileObj.url}));
+      this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: fileObj.url, storageConfig: storageConfig}));
     } else if (fs.existsSync(contract.file)) {
-      this.contractsFiles.push(new File({filename: contract.file, type: File.types.dapp_file, basedir: '', path: contract.file}));
+      this.contractsFiles.push(new File({filename: contract.file, type: File.types.dapp_file, basedir: '', path: contract.file, storageConfig: storageConfig}));
     } else if (fs.existsSync(path.join('./node_modules/', contract.file))) {
-      this.contractsFiles.push(new File({filename: path.join('./node_modules/', contract.file), type: File.types.dapp_file, basedir: '', path: path.join('./node_modules/', contract.file)}));
+      this.contractsFiles.push(new File({filename: path.join('./node_modules/', contract.file), type: File.types.dapp_file, basedir: '', path: path.join('./node_modules/', contract.file), storageConfig: storageConfig}));
     } else {
       this.logger.error(__("contract file not found") + ": " + contract.file);
     }
@@ -422,14 +441,6 @@ Config.prototype.loadEmbarkConfigFile = function() {
   this.embarkConfig = utils.recursiveMerge(configObject, this.embarkConfig);
 
   const contracts = this.embarkConfig.contracts;
-  const newContractsFiles = this.loadFiles(contracts);
-  if (!this.contractFiles || newContractsFiles.length !== this.contractFiles.length || !deepEqual(newContractsFiles, this.contractFiles)) {
-    this.contractsFiles = this.contractsFiles.concat(newContractsFiles).filter((file, index, arr) => {
-      return !arr.some((file2, index2) => {
-        return file.filename === file2.filename && index < index2;
-      });
-    });
-  }
   // determine contract 'root' directories
   this.contractDirectories = contracts.map((dir) => {
     return dir.split("**")[0];
@@ -503,12 +514,13 @@ Config.prototype.loadFiles = function(files) {
   var self = this;
   var originalFiles = utils.filesMatchingPattern(files);
   var readFiles = [];
+  let storageConfig = self.storageConfig;
 
   originalFiles.filter(function(file) {
     return (file[0] === '$' || file.indexOf('.') >= 0);
   }).filter(function(file) {
     let basedir = findMatchingExpression(file, files);
-    readFiles.push(new File({filename: file, type: File.types.dapp_file, basedir: basedir, path: file}));
+    readFiles.push(new File({filename: file, type: File.types.dapp_file, basedir: basedir, path: file, storageConfig: storageConfig}));
   });
 
   var filesFromPlugins = [];
@@ -537,12 +549,12 @@ Config.prototype.loadFiles = function(files) {
 // NOTE: this doesn't work for internal modules
 Config.prototype.loadPluginContractFiles = function() {
   var self = this;
-
+  let storageConfig = self.storageConfig;
   var contractsPlugins = this.plugins.getPluginsFor('contractFiles');
   contractsPlugins.forEach(function(plugin) {
     plugin.contractsFiles.forEach(function(file) {
       var filename = file.replace('./','');
-      self.contractsFiles.push(new File({filename: filename, pluginPath: plugin.pluginPath, type: File.types.custom, path: filename, resolver: function(callback) {
+      self.contractsFiles.push(new File({filename: filename, pluginPath: plugin.pluginPath, type: File.types.custom, path: filename, storageConfig: storageConfig, resolver: function(callback) {
         callback(plugin.loadPluginFile(file));
       }}));
     });

lib/core/file.js

@@ -15,6 +15,8 @@ class File {
     this.pluginPath = options.pluginPath ? options.pluginPath : '';
     this.downloadedImports = false;
     this.importRemappings = []; // mapping downloaded imports to local file
+    this.storageConfig = options.storageConfig;
+    this.ipfsUrl = null;
   }
 
   parseFileForImport(content, isHttpContract, callback) {
@@ -32,8 +34,12 @@ class File {
     const filesToDownload = [];
     const pathWithoutFile = path.dirname(self.path);
     let newContent = content;
+    let storageConfig = self.storageConfig;
+    if (storageConfig && storageConfig.upload && storageConfig.upload.getUrl) {
+        self.ipfsUrl = storageConfig.upload.getUrl;
+    }
     while ((matches = regex.exec(content))) {
-      const httpFileObj = utils.getExternalContractUrl(matches[1]);
+      const httpFileObj = utils.getExternalContractUrl(matches[1],self.ipfsUrl);
       const fileObj = {
         fileRelativePath: path.join(path.dirname(self.filename), matches[1]),
         url: `${pathWithoutFile}/${matches[1]}`
@@ -47,7 +53,7 @@ class File {
       } else if (fs.existsSync(path.join("node_modules", matches[1]))) {
         target = path.join("node_modules", matches[1]);
       }
-      
+
       self.importRemappings.push({
         prefix: matches[1],
         target: fs.dappPath(target)

lib/modules/solidity/index.js

@@ -13,6 +13,8 @@ class Solidity {
     this.solcW = null;
     this.useDashboard = options.useDashboard;
     this.options = embark.config.embarkConfig.options.solc;
+    this.storageConfig = embark.config.storageConfig;
+    this.ipfsUrl = null;
 
     embark.registerCompiler(".sol", this.compile_solidity.bind(this));
 
@@ -69,7 +71,12 @@ class Solidity {
         if (self.solcAlreadyLoaded) {
           return callback();
         }
-        self.solcW = new SolcW(self.embark, {logger: self.logger, events: self.events, ipc: self.ipc, useDashboard: self.useDashboard});
+        let storageConfig = self.storageConfig;
+        if (storageConfig && storageConfig.upload && storageConfig.upload.getUrl) {
+            self.ipfsUrl = storageConfig.upload.getUrl;
+        }
+        self.solcW = new SolcW(self.embark, {logger: self.logger, events: self.events, ipc: self.ipc, useDashboard: self.useDashboard, ipfsUrl: self.ipfsUrl});
+
 
         self.logger.info(__("loading solc compiler") + "..");
         self.solcW.load_compiler(function (err) {

lib/modules/solidity/solcP.js

@@ -13,11 +13,12 @@ class SolcProcess extends ProcessWrapper {
     super({pingParent: false});
     this._logger = options.logger;
     this._showSpinner = options.showSpinner === true;
+    this._ipfsUrl = options.ipfsUrl;
   }
 
   findImports(filename) {
     if (filename.startsWith('http') || filename.startsWith('git')) {
-      const fileObj = Utils.getExternalContractUrl(filename);
+      const fileObj = Utils.getExternalContractUrl(filename,this._ipfsUrl);
       filename = fileObj.filePath;
     }
     if (fs.existsSync(filename)) {
@@ -47,7 +48,7 @@ class SolcProcess extends ProcessWrapper {
         if(timer) timer.end();
         resolve();
       }).catch(reject);
-      
+
     });
   }
 
@@ -88,4 +89,3 @@ process.on('message', (msg) => {
     });
   }
 });
-

lib/modules/solidity/solcW.js

@@ -14,6 +14,7 @@ class SolcW {
     this.compilerLoaded = false;
     this.solcProcess = null;
     this.useDashboard = options.useDashboard;
+    this.ipfsUrl = options.ipfsUrl;
   }
 
   load_compiler(done) {
@@ -45,6 +46,7 @@ class SolcW {
       modulePath: utils.joinPath(__dirname, 'solcP.js'),
       logger: self.logger,
       events: self.events,
+      ipfsUrl: self.ipfsUrl,
       silent: false
     });
 
@@ -52,7 +54,7 @@ class SolcW {
       this.events.request("version:get:solc", function(solcVersion) {
         if (solcVersion === currentSolcVersion) {
           return self.solcProcess.send({action: 'loadCompiler', requirePath: 'solc'});
-        } 
+        }
         self.events.request("version:getPackagePath", "solc", solcVersion, function(err, path) {
           if (err) {
             return done(err);
@@ -98,4 +100,3 @@ class SolcW {
 }
 
 module.exports = SolcW;
-

lib/utils/utils.js

@@ -224,11 +224,13 @@ function proposeAlternative(word, _dictionary, _exceptions) {
   return propose(word, dictionary, {threshold: 0.3});
 }
 
-function getExternalContractUrl(file) {
+function getExternalContractUrl(file,ipfsUrl) {
   const constants = require('../constants');
   let url;
   const RAW_URL = 'https://raw.githubusercontent.com/';
   const MALFORMED_ERROR = 'Malformed Github URL for ';
+  const MALFORMED_IPFS_ERROR = 'Malformed IPFS URL for ';
+  const IPFS_GETURL_NOTAVAILABLE = 'IPFS getUrl is not available. Please set it in your storage config. For more info: https://embark.status.im/docs/storage_configuration.html';
   if (file.startsWith('https://github')) {
     const match = file.match(/https:\/\/github\.[a-z]+\/(.*)/);
     if (!match) {
@@ -236,6 +238,28 @@ function getExternalContractUrl(file) {
       return null;
     }
     url = `${RAW_URL}${match[1].replace('blob/', '')}`;
+  } else if (file.startsWith('ipfs')) {
+    if(!ipfsUrl) {
+      console.error(IPFS_GETURL_NOTAVAILABLE);
+      return null;
+    }
+    let match = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)\/(.*)/);
+    if(!match) {
+      match = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)/);
+      if(!match) {
+        console.error(MALFORMED_IPFS_ERROR + file);
+        return null;
+      }
+    }
+    let matchResult = match[1];
+    if(match[2]) {
+      matchResult += '/' + match[2];
+    }
+    url = `${ipfsUrl}${matchResult}`;
+    return {
+      url,
+      filePath: constants.httpContractsDirectory + matchResult
+    };
   } else if (file.startsWith('git')) {
     // Match values
     // [0] entire input

test/config.js

@@ -148,7 +148,9 @@ describe('embark.Config', function () {
           "importRemappings": [],
           "basedir": "",
           "resolver": undefined,
-          "downloadedImports": false
+          "downloadedImports": false,
+          "storageConfig": null,
+          "ipfsUrl": null
         },
         {
           "filename": ".embark/contracts/status-im/contracts/master/contracts/identity/ERC725.sol",
@@ -158,7 +160,9 @@ describe('embark.Config', function () {
           "importRemappings": [],
           "basedir": "",
           "resolver": undefined,
-          "downloadedImports": false
+          "downloadedImports": false,
+          "storageConfig": null,
+          "ipfsUrl": null
         }
       ];
       config.loadExternalContractsFiles();