From 0cb79d84a0ad46ef302e72ff4697ea566e293403 Mon Sep 17 00:00:00 2001 From: emizzle Date: Wed, 30 May 2018 16:34:36 +1000 Subject: [PATCH] Dynamic selection of storage provider now working based on improved storage config. swarm plugin now re-initialises the bzz object when it's availability is checked. this creates a much more stable swarm implementation on the dapp side. surrounded the storage provider init code block with embark env ready added alternate swarm gateway url can now upload dapp to ipfs and run swarm storage and vice versa --- js/embark.js | 12 ------ lib/cmd.js | 2 +- lib/core/engine.js | 1 - lib/i18n/locales/en.json | 5 +++ lib/index.js | 3 +- lib/modules/ipfs/embarkjs.js | 25 ++++++------ lib/modules/ipfs/index.js | 18 +++------ lib/modules/storage/embarkjs.js | 20 +++++---- lib/modules/storage/index.js | 56 +++++++++++++++++++------- lib/modules/swarm/embarkjs.js | 52 +++++++++++++++--------- lib/modules/swarm/index.js | 45 ++++++++------------- lib/modules/swarm/upload.js | 5 ++- test_apps/test_app/config/storage.json | 6 +-- 13 files changed, 135 insertions(+), 115 deletions(-) diff --git a/js/embark.js b/js/embark.js index d86617664..45d22c333 100644 --- a/js/embark.js +++ b/js/embark.js @@ -230,18 +230,6 @@ EmbarkJS.Storage.setProvider = function(provider, options) { return providerObj.setProvider(options); }; -EmbarkJS.Storage.setProviders = function(provider, dappConnOptions) { - let providerObj = this.Providers[provider]; - - if (!providerObj) { - throw new Error('Unknown storage provider'); - } - - this.currentStorage = providerObj; - - return providerObj.setProviders(dappConnOptions); -}; - EmbarkJS.Storage.isAvailable = function(){ if (!this.currentStorage) { throw new Error('Storage provider not set; e.g EmbarkJS.Storage.setProvider("ipfs")'); diff --git a/lib/cmd.js b/lib/cmd.js index 2550e48fa..0e5c51996 100644 --- a/lib/cmd.js +++ b/lib/cmd.js @@ -203,7 +203,7 @@ class Cmd { .option('--loglevel [loglevel]', __('level of logging to display') + ' ["error", "warn", "info", "debug", "trace"]', /^(error|warn|info|debug|trace)$/i, 'debug') .option('--locale [locale]', __('language to use (default: en)')) .option('-c, --client [client]', __('Use a specific ethereum client or simulator (supported: %s)', 'geth, testrpc')) - .description(__('Upload your dapp to a decentralized storage') + ' (e.g embark upload ipfs).') + .description(__('Upload your dapp to a decentralized storage') + '.') .action(function (env, _options) { i18n.setOrDetectLocale(_options.locale); _options.env = env || 'development'; diff --git a/lib/core/engine.js b/lib/core/engine.js index 41be4a4c7..78b125198 100644 --- a/lib/core/engine.js +++ b/lib/core/engine.js @@ -135,7 +135,6 @@ class Engine { plugins: this.plugins }); this.events.on('code-generator-ready', function () { - console.log('CODE GENERATOR READY EVENT FIRED'); self.events.request('code', function (abi, contractsJSON) { pipeline.build(abi, contractsJSON, null, () => { if (self.watch) { diff --git a/lib/i18n/locales/en.json b/lib/i18n/locales/en.json index fb02432dd..e81ff233e 100644 --- a/lib/i18n/locales/en.json +++ b/lib/i18n/locales/en.json @@ -132,6 +132,11 @@ "IPFS node detected": "IPFS node detected", "Webserver is offline": "Webserver is offline", "DApp path length is too long: \"": "DApp path length is too long: \"", + "This is known to cause issues with some applications, please consider reducing your DApp path's length to 66 characters or less.": "This is known to cause issues with some applications, please consider reducing your DApp path's length to 66 characters or less.", + "deploying to swarm!": "deploying to swarm!", + "adding %s to swarm": "adding %s to swarm", + "successfully uploaded to swarm": "successfully uploaded to swarm", + "Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}{{port}}.": "Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}{{port}}." "This is known to cause issues with some applications, please consider reducing your DApp path's length to 66 characters or less.": "This is known to cause issues with some applications, please consider reducing your DApp path's length to 66 characters or less." "no contracts found": "no contracts found", "DApp path length is too long: \"": "DApp path length is too long: \"", diff --git a/lib/index.js b/lib/index.js index aa218577d..9ee88d17d 100644 --- a/lib/index.js +++ b/lib/index.js @@ -397,7 +397,8 @@ class Embark { checkFn.fn(function (serviceCheckResult) { if (!serviceCheckResult.status || serviceCheckResult.status === 'off') { let config = engine.config.storageConfig.upload; - return callback({message: __('Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}:{{port}}.', {platform: platform, protocol: config.protocol, host: config.host, port: config.port})}); + let port = config.port ? ':' + config.port : ''; + return callback({message: __('Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}{{port}}.', {platform: platform, protocol: config.protocol, host: config.host, port: port})}); } callback(); }); diff --git a/lib/modules/ipfs/embarkjs.js b/lib/modules/ipfs/embarkjs.js index 39c849eda..82bf19324 100644 --- a/lib/modules/ipfs/embarkjs.js +++ b/lib/modules/ipfs/embarkjs.js @@ -7,7 +7,8 @@ __embarkIPFS.setProvider = function (options) { var promise = new Promise(function (resolve, reject) { try { if (options === undefined) { - self.ipfsConnection = IpfsApi('localhost', '5001'); + self._config = options; + self._ipfsConnection = IpfsApi('localhost', '5001'); self._getUrl = "http://localhost:8080/ipfs/"; } else { var ipfsOptions = {host: options.host || options.server, protocol: 'http'}; @@ -17,13 +18,13 @@ __embarkIPFS.setProvider = function (options) { if (options.port && options.port !== 'false') { ipfsOptions.port = options.port; } - self.ipfsConnection = IpfsApi(ipfsOptions); + self._ipfsConnection = IpfsApi(ipfsOptions); self._getUrl = options.getUrl || "http://localhost:8080/ipfs/"; } resolve(self); } catch (err) { console.error(err); - self.ipfsConnection = null; + self._ipfsConnection = null; reject(new Error('Failed to connect to IPFS')); } }); @@ -33,11 +34,11 @@ __embarkIPFS.setProvider = function (options) { __embarkIPFS.saveText = function (text) { const self = this; var promise = new Promise(function (resolve, reject) { - if (!self.ipfsConnection) { + if (!self._ipfsConnection) { var connectionError = new Error('No IPFS connection. Please ensure to call Embark.Storage.setProvider()'); reject(connectionError); } - self.ipfsConnection.add(self.ipfsConnection.Buffer.from(text), function (err, result) { + self._ipfsConnection.add(self._ipfsConnection.Buffer.from(text), function (err, result) { if (err) { reject(err); } else { @@ -54,11 +55,11 @@ __embarkIPFS.get = function (hash) { // TODO: detect type, then convert if needed //var ipfsHash = web3.toAscii(hash); var promise = new Promise(function (resolve, reject) { - if (!self.ipfsConnection) { + if (!self._ipfsConnection) { var connectionError = new Error('No IPFS connection. Please ensure to call Embark.Storage.setProvider()'); reject(connectionError); } - self.ipfsConnection.get(hash, function (err, files) { + self._ipfsConnection.get(hash, function (err, files) { if (err) { return reject(err); } @@ -78,15 +79,15 @@ __embarkIPFS.uploadFile = function (inputSelector) { } var promise = new Promise(function (resolve, reject) { - if (!self.ipfsConnection) { + if (!self._ipfsConnection) { var connectionError = new Error('No IPFS connection. Please ensure to call Embark.Storage.setProvider()'); reject(connectionError); } var reader = new FileReader(); reader.onloadend = function () { var fileContent = reader.result; - var buffer = self.ipfsConnection.Buffer.from(fileContent); - self.ipfsConnection.add(buffer, function (err, result) { + var buffer = self._ipfsConnection.Buffer.from(fileContent); + self._ipfsConnection.add(buffer, function (err, result) { if (err) { reject(err); } else { @@ -102,10 +103,10 @@ __embarkIPFS.uploadFile = function (inputSelector) { __embarkIPFS.isAvailable = function () { return new Promise((resolve) => { - if (!this.ipfsConnection) { + if (!this._ipfsConnection) { return resolve(false); } - this.ipfsConnection.id() + this._ipfsConnection.id() .then((id) => { resolve(Boolean(id)); }) diff --git a/lib/modules/ipfs/index.js b/lib/modules/ipfs/index.js index 1f9217005..722e44d3c 100644 --- a/lib/modules/ipfs/index.js +++ b/lib/modules/ipfs/index.js @@ -1,7 +1,7 @@ -let UploadIPFS = require('./upload.js'); -let utils = require('../../utils/utils.js'); -let fs = require('../../core/fs.js'); -let IpfsApi = require('ipfs-api'); +const UploadIPFS = require('./upload.js'); +const utils = require('../../utils/utils.js'); +const fs = require('../../core/fs.js'); +const IpfsApi = require('ipfs-api'); const _ = require('underscore'); class IPFS { @@ -16,18 +16,12 @@ class IPFS { this.protocol = options.protocol || this.storageConfig.upload.protocol; this.addCheck = options.addCheck; this.embark = embark; - - // this.commandlineDeploy(); - // this.setServiceCheck(); - // this.addProviderToEmbarkJS(); - // this.addSetProvider(); - // this.addObjectToConsole(); } commandlineDeploy() { let upload_ipfs = new UploadIPFS({ buildDir: this.buildDir || 'dist/', - storageConfig: this.storageConfig, + storageConfig: this.storageConfig.upload, configIpfsBin: this.storageConfig.ipfs_bin || "ipfs" }); @@ -126,7 +120,7 @@ class IPFS { // } addObjectToConsole() { - let ipfs = IpfsApi(this.storageConfig.host, this.storageConfig.port); + let ipfs = IpfsApi(this.host, this.port); this.events.emit("runcode:register", "ipfs", ipfs); } diff --git a/lib/modules/storage/embarkjs.js b/lib/modules/storage/embarkjs.js index dac274815..8f7c41513 100644 --- a/lib/modules/storage/embarkjs.js +++ b/lib/modules/storage/embarkjs.js @@ -5,20 +5,25 @@ import {findSeries} from 'p-iteration'; let __embarkStorage = {}; __embarkStorage.setProviders = async function (dappConnOptions) { - var self = this; try { let workingConnFound = await findSeries(dappConnOptions, async (dappConn) => { if(dappConn === '$BZZ' || dappConn.provider === 'swarm'){ - let options = dappConnOptions; - options.useOnlyCurrentProvider = dappConn === '$BZZ'; - await EmbarkJS.Storage.setProvider('swarm', options); - return EmbarkJS.Storage.isAvailable(); + let options = dappConn; + if(dappConn === '$BZZ') options = {"useOnlyGivenProvider": true}; + try{ + await EmbarkJS.Storage.setProvider('swarm', options); + let isAvailable = await EmbarkJS.Storage.isAvailable(); + return isAvailable; + }catch(err){ + return false; // catch errors for when bzz object not initialised but config has requested it to be used + } } else if(dappConn.provider === 'ipfs') { // set the provider then check the connection, if true, use that provider, else, check next provider try{ await EmbarkJS.Storage.setProvider('ipfs', dappConn); - return EmbarkJS.Storage.isAvailable(); + let isAvailable = await EmbarkJS.Storage.isAvailable(); + return isAvailable; } catch(err) { return false; } @@ -26,7 +31,6 @@ __embarkStorage.setProviders = async function (dappConnOptions) { }); if(!workingConnFound) throw new Error('Could not connect to a storage provider using any of the dappConnections in the storage config'); } catch (err) { - self.ipfsConnection = null; - throw new Error('Failed to connect to IPFS: ' + err.message); + throw new Error('Failed to connect to a storage provider: ' + err.message); } }; diff --git a/lib/modules/storage/index.js b/lib/modules/storage/index.js index af67d58c0..1c878cb3e 100644 --- a/lib/modules/storage/index.js +++ b/lib/modules/storage/index.js @@ -5,28 +5,52 @@ let _ = require('underscore'); class Storage { constructor(embark, options){ + this._embark = embark; this._storageConfig = options.storageConfig; let storageProviderCls = require(`../${this._storageConfig.upload.provider}/index.js`); - this._uploadProvider = new storageProviderCls(embark, this._storageConfig); /*eslint no-new: "off"*/ + let uploadProvider = new storageProviderCls(embark, options); /*eslint no-new: "off"*/ - if(typeof this._uploadProvider.initProvider == 'function') this._uploadProvider.initProvider(); - if(typeof this._uploadProvider.commandlineDeploy == 'function') this._uploadProvider.commandlineDeploy(); - if(typeof this._uploadProvider.setServiceCheck == 'function') this._uploadProvider.setServiceCheck(); - if(typeof this._uploadProvider.addObjectToConsole == 'function') this._uploadProvider.addObjectToConsole(); + if(typeof uploadProvider.commandlineDeploy == 'function') uploadProvider.commandlineDeploy(); + if(typeof uploadProvider.setServiceCheck == 'function') uploadProvider.setServiceCheck(); + if(typeof uploadProvider.addObjectToConsole == 'function') uploadProvider.addObjectToConsole(); - // loop through all available providers and add "register provider" code in EmbarkJS - // which allows the provider to be set in the DApp + // loop through all available providers and add the provider code to embarkjs this._storageConfig.available_providers.forEach(providerStr => { - let storageProviderCls = require(`../${providerStr}/index.js`); - this._storageProvider = new storageProviderCls(this.storageConfig); /*eslint no-new: "off"*/ - if(typeof this._storageProvider.addProviderToEmbarkJS == 'function') this._storageProvider.addProviderToEmbarkJS(); + let storageProvider; + + // check if we've already instantiated our storage class and reuse + if(providerStr === this._storageConfig.upload.provider){ + storageProvider = uploadProvider; + } + // otherwise instantiate the storage provider + else { + let storageProviderCls = require(`../${providerStr}/index.js`); + storageProvider = new storageProviderCls(embark, options); /*eslint no-new: "off"*/ + } + + // add __embarkSwarm and __embarkIPFS objects to EmbarkJS + if(typeof storageProvider.addProviderToEmbarkJS == 'function') storageProvider.addProviderToEmbarkJS(); }); - // add the code to call setProviders in embarkjs + // add the storage provider code (__embarkStorage) to embarkjs + this.addProviderToEmbarkJS(); + + // add the code to call setProviders in embarkjs after embark is ready this.addSetProviders(); } + addProviderToEmbarkJS(){ + // TODO: make this a shouldAdd condition + if (this._storageConfig === {} || !this._storageConfig.dappConnection || !this._storageConfig.dappConnection.length) { + return; + } + + let code = "\n" + fs.readFileSync(utils.joinPath(__dirname, 'embarkjs.js')).toString(); + + this._embark.addCodeToEmbarkJS(code); + } + addSetProviders() { // TODO: make this a shouldAdd condition if (this._storageConfig === {} || !this._storageConfig.dappConnection || !this._storageConfig.dappConnection.length) { @@ -39,11 +63,13 @@ class Storage { return _.contains(this._storageConfig.available_providers, conn.provider) || (conn === '$BZZ' && hasSwarm); }); - let code = ""; - code += "\n" + fs.readFileSync(utils.joinPath(__dirname, 'embarkjs.js')).toString(); - code += `\n__embarkStorage.setProviders(${JSON.stringify(connectionsToSet)}));`; + let code = `\n__embarkStorage.setProviders(${JSON.stringify(connectionsToSet)});`; + + let shouldInit = (storageConfig) => { + return (connectionsToSet !== undefined && connectionsToSet.length > 0 && storageConfig.enabled === true); + }; - this.embark.addCodeToEmbarkJS(code); + this._embark.addProviderInit('storage', code, shouldInit); } } diff --git a/lib/modules/swarm/embarkjs.js b/lib/modules/swarm/embarkjs.js index 18ed0b08e..7f5dc4a5e 100644 --- a/lib/modules/swarm/embarkjs.js +++ b/lib/modules/swarm/embarkjs.js @@ -1,37 +1,49 @@ -/*global web3, some */ +/*global web3 */ let __embarkSwarm = {}; const bytes = require("eth-lib/lib/bytes"); __embarkSwarm.setProvider = function (options) { - this.bzz = web3.bzz; - this.protocol = options.protocol || 'http'; - this.connectUrl = `${this.protocol}://${options.host}:${options.port}`; - this.connectError = new Error(`Cannot connect to Swarm node on ${this.connectUrl}`); - this.useOnlyCurrentProvider = options.useOnlyCurrentProvider; - //this._getUrl = options.getUrl || `${this.connectUrl}/bzzr:/`; + let protocol = options.protocol || 'http'; + let port = options.port ? `:${options.port}` : ''; + + this._config = options; + this._connectUrl = `${protocol}://${options.host}${port}`; + this._connectError = new Error(`Cannot connect to Swarm node on ${this._connectUrl}`); return new Promise((resolve, reject) => { try { - if (!this.bzz.currentProvider && !this.useOnlyCurrentProvider) { - this.bzz.setProvider(this.connectUrl); + if (!web3.bzz.currentProvider && !options.useOnlyGivenProvider) { + web3.bzz.setProvider(this._connectUrl); + } + else if(options.useOnlyGivenProvider && web3.bzz.givenProvider !== null){ + web3.bzz.setProvider(web3.bzz.givenProvider); } resolve(this); } catch (err) { console.log(err); - reject(this.connectError); + reject(this._connectError); } }); }; __embarkSwarm.isAvailable = function () { return new Promise((resolve, reject) => { - if (!this.bzz) { + // if web3 swarm object doesn't exist + if (!web3.bzz) { return resolve(false); } - this.bzz.isAvailable() + // swarm obj exists, but has no provider set (seems to happen a LOT!), + // try setting the provider to our currently set provider again + else if(!web3.bzz.currentProvider && this._config.host){ + web3.bzz.setProvider(this._connectUrl); + } + if (!web3.bzz.currentProvider) { + return resolve(false); + } + web3.bzz.isAvailable() .then(resolve) .catch(() => { - reject(this.connectError); + reject(this._connectError); }); }); }; @@ -40,9 +52,9 @@ __embarkSwarm.saveText = function (text) { return new Promise((resolve, reject) => { this.isAvailable().then((isAvailable) => { if (!isAvailable) { - return reject(this.connectError); + return reject(this._connectError); } - this.bzz.upload(text) + web3.bzz.upload(text) .then(resolve) .catch(reject); }).catch(reject); @@ -53,9 +65,9 @@ __embarkSwarm.get = function (hash) { return new Promise((resolve, reject) => { this.isAvailable().then((isAvailable) => { if (!isAvailable) { - return reject(this.connectError); + return reject(this._connectError); } - this.bzz.download(hash) + web3.bzz.download(hash) .then((uint8Array) => resolve(bytes.toString(bytes.fromUint8Array(uint8Array)))) .catch(reject); }).catch(reject); @@ -75,9 +87,9 @@ __embarkSwarm.uploadFile = function (inputSelector) { const fileContent = new Uint8Array(event.target.result); this.isAvailable().then((isAvailable) => { if (!isAvailable) { - return reject(this.connectError); + return reject(this._connectError); } - this.bzz.upload(fileContent) + web3.bzz.upload(fileContent) .then(resolve) .catch(reject); }).catch(reject); @@ -88,6 +100,6 @@ __embarkSwarm.uploadFile = function (inputSelector) { }; __embarkSwarm.getUrl = function (hash) { - return this._getUrl + hash; + return `${this._config.getUrl || this._connectUrl + '/bzzr:/'}${hash}`; }; diff --git a/lib/modules/swarm/index.js b/lib/modules/swarm/index.js index b14b2939a..b40d61c83 100644 --- a/lib/modules/swarm/index.js +++ b/lib/modules/swarm/index.js @@ -1,6 +1,7 @@ -let UploadSwarm = require('./upload.js'); -let utils = require('../../utils/utils.js'); -let fs = require('../../core/fs.js'); +const UploadSwarm = require('./upload.js'); +const utils = require('../../utils/utils.js'); +const fs = require('../../core/fs.js'); +const Web3Bzz = require('web3-bzz'); class Swarm { @@ -9,30 +10,26 @@ class Swarm { this.events = embark.events; this.buildDir = options.buildDir; this.storageConfig = options.storageConfig; - this.host = options.host || options.storageConfig.upload.host; - this.port = options.port || options.storageConfig.upload.port; - this.protocol = options.protocol || options.storageConfig.upload.protocol; this.addCheck = options.addCheck; this.embark = embark; - this.bzz = options.bzz; - - // this.initProvider(); - // this.commandlineDeploy(); - // this.setServiceCheck(); - // this.addProviderToEmbarkJS(); - // this.addSetProvider(); - } - - initProvider(){ - if(!this.bzz.currentProvider) { - this.bzz.setProvider(`${this.protocol}://${this.host}:${this.port}`); - } + + let host = options.host || options.storageConfig.upload.host; + let port = options.port || options.storageConfig.upload.port; + if(port) port = ':' + port; + else port = ''; + let protocol = options.protocol || options.storageConfig.upload.protocol || 'http'; + this.providerUrl = `${protocol}://${host}${port}`; + this.getUrl = options.storageConfig.upload.getUrl || this.providerUrl + '/bzzr:/'; + + this.bzz = new Web3Bzz(this.providerUrl); } commandlineDeploy() { this.upload_swarm = new UploadSwarm({ buildDir: this.buildDir || 'dist/', storageConfig: this.storageConfig, + connectUrl: this.providerUrl, + getUrl: this.getUrl, bzz: this.bzz }); @@ -102,16 +99,6 @@ class Swarm { this.embark.addCodeToEmbarkJS(code); } - - // addSetProvider() { - // let code = "\nEmbarkJS.Storage.setProviders('swarm'," + JSON.stringify(this.storageConfig.dappConnection) + ");"; - - // let shouldInit = (storageConfig) => { - // return (this.storageConfig.dappConnection !== undefined && this.storageConfig.dappConnection.some((dappConn) => dappConn.provider === 'swarm') && storageConfig.enabled === true); - // }; - - // this.embark.addProviderInit('storage', code, shouldInit); - // } } module.exports = Swarm; diff --git a/lib/modules/swarm/upload.js b/lib/modules/swarm/upload.js index 4defd9118..607e361d3 100644 --- a/lib/modules/swarm/upload.js +++ b/lib/modules/swarm/upload.js @@ -7,6 +7,8 @@ class Swarm { this.buildDir = options.buildDir || 'dist/'; this.bzz = options.bzz; this.storageConfig = options.storageConfig; + this.providerUrl = options.providerUrl; + this.getUrl = options.getUrl; } deploy() { @@ -31,7 +33,8 @@ class Swarm { if (!dir_hash) { return callback('No directory hash was returned'); } - console.log(("=== " + __("DApp available at") + `${self.storageConfig.getUrl}${dir_hash}/`).green); + console.log(("=== " + __("DApp available at") + ` ${self.getUrl}${dir_hash}/`).green); + console.log(("=== " + __("DApp available at") + ` http://swarm-gateways.net/bzzr:/${dir_hash}`).green); callback(); } diff --git a/test_apps/test_app/config/storage.json b/test_apps/test_app/config/storage.json index 6d06e3c74..139121561 100644 --- a/test_apps/test_app/config/storage.json +++ b/test_apps/test_app/config/storage.json @@ -20,10 +20,10 @@ "development": { "enabled": true, "upload": { - "provider": "ipfs", + "provider": "swarm", "host": "localhost", - "port": 5001, - "getUrl": "http://localhost:8080/ipfs/" + "port": 8500, + "getUrl": "http://localhost:8500/bzzr:/" } }, "livenet": {