embarklabs
/
embark
mirror of https://github.com/embarklabs/embark.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Further fixes for starting dappConnection processes automatically 

moved storage process launching to storage module

refactored all cases of storage url building to util methods

updated cors so that a processes will not include itself in cors list

changed child process command console output to trace, so it's visible when --logLevel == trace
This commit is contained in:
emizzle 2018-06-01 13:12:17 +10:00
parent 927f562a41
commit a11047c681
11 changed files with 266 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/cmds/blockchain/blockchain.js
View File

@ -120,7 +120,7 @@ Blockchain.prototype.run = function() {
return;
}
args = _.compact(args);
console.log(`RUNNING GETH COMMAND: ${cmd} ${JSON.stringify(args)}`);
console.trace(`Geth command: ${cmd} ${args.join(' ')}`);
self.child = child_process.spawn(cmd, args, {cwd: process.cwd()});
self.child.on('error', (err) => {

6
lib/core/config.js
View File

@ -81,8 +81,7 @@ Config.prototype._updateBlockchainCors = function(){
let corsParts = [];
if(webServerConfig && webServerConfig.enabled) {
let webServerPort = webServerConfig.port ? `:${webServerConfig.port}` : '';
if(webServerConfig.host) corsParts.push(`http://${webServerConfig.host}${webServerPort}`);
if(webServerConfig.host) corsParts.push(utils.buildUrlFromConfig(webServerConfig));
}
if(storageConfig && storageConfig.enabled) {
// if getUrl is specified in the config, that needs to be included in cors
@ -95,8 +94,7 @@ Config.prototype._updateBlockchainCors = function(){
}
// use our modified getUrl or in case it wasn't specified, use a built url
else{
let port = storageConfig.upload.port ? `:${storageConfig.upload.port}` : '';
corsParts.push(`${storageConfig.upload.protocol || 'http'}://${storageConfig.upload.host}${port}`);
corsParts.push(utils.buildUrlFromConfig(storageConfig.upload));
}
}

9
lib/core/engine.js
View File

@ -265,8 +265,15 @@ class Engine {
storageService(_options) {
this.registerModule('storage', {
addCheck: this.servicesMonitor.addCheck.bind(this.servicesMonitor),
storageConfig: this.config.storageConfig,
webServerConfig: this.config.webServerConfig,
blockchainConfig: this.config.blockchainConfig,
host: _options.host,
port: _options.port
port: _options.port,
servicesMonitor: this.servicesMonitor,
events: this.events,
logger: this.logger,
context: this.context
});
}

4
lib/i18n/locales/en.json
View File

@ -167,5 +167,7 @@
"Storage process for swarm ended before the end of this process. Code: 1": "Storage process for swarm ended before the end of this process. Code: 1",
"Storage process for ipfs ended before the end of this process. Code: 12": "Storage process for ipfs ended before the end of this process. Code: 12",
"Blockchain process ended before the end of this process. Code: %s": "Blockchain process ended before the end of this process. Code: %s",
"Cannot start {{platform}} node on {{protocol}}://{{host}}{{port}}.": "Cannot start {{platform}} node on {{protocol}}://{{host}}{{port}}."
"Cannot start {{platform}} node on {{protocol}}://{{host}}{{port}}.": "Cannot start {{platform}} node on {{protocol}}://{{host}}{{port}}.",
"Cannot upload: {{platform}} node is not running on {{url}}.": "Cannot upload: {{platform}} node is not running on {{url}}.",
"Cannot start {{platform}} node on {{url}}.": "Cannot start {{platform}} node on {{url}}."
}

126
lib/index.js
View File

@ -1,8 +1,5 @@
let async = require('async');
const constants = require('./constants');
const _ = require('underscore');
const StorageProcessesLauncher = require('./processes/storageProcesses/storageProcessesLauncher');
// require("./utils/debug_util.js")(__filename, async);
require('colors');
@ -67,43 +64,6 @@ class Embark {
templateGenerator.generate(destinationFolder, name);
}
_checkStorageEndpoint(engine, platform, callback) {
let checkFn;
_.find(engine.servicesMonitor.checkList, (value, key) => {
if(key.toLowerCase() === platform.toLowerCase()){
checkFn = value;
return true;
}
});
if (!checkFn || typeof checkFn.fn !== 'function') {
return callback();
}
checkFn.fn(function (serviceCheckResult) {
if (!serviceCheckResult.status || serviceCheckResult.status === 'off') {
return callback('No node');
}
callback();
});
}
_startStorageNode(engine, platform, callback) {
const storageProcessesLauncher = new StorageProcessesLauncher({
logger: engine.logger,
events: engine.events,
storageConfig: engine.config.storageConfig,
webServerConfig: engine.config.webServerConfig,
blockchainConfig: engine.config.blockchainConfig
});
return storageProcessesLauncher.launchProcess(platform.toLowerCase(), (err) => {
if (err) {
engine.logger.error(err);
return callback(err);
}
callback();
});
}
run(options) {
let self = this;
self.context = options.context || [constants.contexts.run, constants.contexts.build];
@ -172,20 +132,6 @@ class Embark {
});
});
// Check storage
const platform = engine.config.storageConfig.provider;
self._checkStorageEndpoint(engine, platform, (err) => {
if (!err) {
return;
}
self._startStorageNode(engine, platform, (err) => {
if (err) {
engine.logger.error('Error while starting a storage process for ' + platform);
engine.logger.error(err);
}
});
});
engine.events.on('outputDone', function () {
engine.logger.info((__("Looking for documentation? You can find it at") + " ").cyan + "http://embark.status.im/docs/".green.underline + ".".cyan);
engine.logger.info(__("Ready").underline);
@ -334,7 +280,6 @@ class Embark {
}
upload(options) {
const self = this;
this.context = options.context || [constants.contexts.upload, constants.contexts.build];
let engine = new Engine({
@ -350,7 +295,8 @@ class Embark {
events: options.events,
logger: options.logger,
config: options.config,
plugins: options.plugins
plugins: options.plugins,
context: this.context
});
engine.init();
@ -371,74 +317,6 @@ class Embark {
engine.startMonitor();
callback();
},
function checkStorageService(callback){
let config = engine.config.storageConfig.upload;
let port = config.port ? ':' + config.port : '';
const errorObj = {message: __('Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}{{port}}.', {platform: platform, protocol: config.protocol || 'http', host: config.host, port: port})};
// start the upload storage node
self._checkStorageEndpoint(engine, platform, function (err) {
if (!err) {
return callback();
}
self._startStorageNode(engine, platform, (err) => {
if (err) {
engine.logger.error(err);
return callback(errorObj);
}
// Check endpoint again to see if really did start
self._checkStorageEndpoint(engine, platform, (err) => {
if (err) {
return callback(errorObj);
}
callback();
});
});
});
if (!checkFn || typeof checkFn.fn !== 'function') {
return callback();
}
checkFn.fn(function (serviceCheckResult) {
if (!serviceCheckResult.status || serviceCheckResult.status === 'off') {
let config = engine.config.storageConfig.upload;
let port = config.port ? ':' + config.port : '';
return callback({message: __('Cannot upload: {{platform}} node is not running on {{protocol}}://{{host}}{{port}}.', {platform: platform, protocol: config.protocol, host: config.host, port: port})});
}
callback();
});
},
function checkDappConnectionStorageService(callback){
// start any dappConnection storage nodes
engine.config.storageConfig.dappConnection.forEach(dappConn => {
if(!dappConn.provider || dappConn.provider === platform) return; // don't start the process we've just started above
let port = dappConn.port ? ':' + dappConn.port : '';
const errorObj = {message: __('Cannot start {{platform}} node on {{protocol}}://{{host}}{{port}}.', {platform: dappConn.provider, protocol: dappConn.protocol || 'http', host: dappConn.host, port: port})};
self._checkStorageEndpoint(engine, dappConn.provider, function (err) {
console.log(`ERROR RETURNED FROM CHECK FOR ${dappConn.provider}: ${err}`);
if (!err) {
return callback();
}
self._startStorageNode(engine, dappConn.provider, (err) => {
console.log('STARTING PROCESS FOR ' + dappConn.provider);
if (err) {
console.log('ERROR STARTING node ' + err);
engine.logger.error(err);
return callback(errorObj);
}
// Check endpoint again to see if really did start
self._checkStorageEndpoint(engine, dappConn.provider, (err) => {
if (err) {
console.log('ERROR STARTING node 2 ' + err);
return callback(errorObj);
}
callback();
});
});
});
});
},
function setupStoragePlugin(callback){
let pluginList = engine.plugins.listPlugins();
if (pluginList.length > 0) {

196
lib/modules/storage/index.js
View File

@ -1,34 +1,115 @@
let utils = require('../../utils/utils.js');
let fs = require('../../core/fs.js');
let _ = require('underscore');
const utils = require('../../utils/utils.js');
const fs = require('../../core/fs.js');
const _ = require('underscore');
const async = require('async');
const StorageProcessesLauncher = require('../../processes/storageProcesses/storageProcessesLauncher');
const constants = require('../../constants');
class Storage {
constructor(embark, options){
this._embark = embark;
this._options = options;
this._storageConfig = options.storageConfig;
this._webServerConfig = options.webServerConfig;
this._blockchainConfig = options.blockchainConfig;
this._servicesMonitor = options.servicesMonitor;
this._events = options.events;
this._logger = options.logger;
// filter list of dapp connections based on available_providers set in config
let hasSwarm = _.contains(this._storageConfig.available_providers, 'swarm'); // don't need to eval this in every loop iteration
// contains valid dapp storage providers
this._validDappProviders = _.filter(this._storageConfig.dappConnection, (conn) => {
return _.contains(this._storageConfig.available_providers, conn.provider) || (conn === '$BZZ' && hasSwarm);
});
this.initStorageForEmbark();
this.initStorageForDapp();
// don't start storage processes on build command, only on upload or run
if(_.contains(options.context, constants.contexts.upload) || _.contains(options.context, constants.contexts.run)){
this.startStorageProcesses();
}
}
_checkStorageEndpoint(platform, callback) {
let checkFn;
let self = this;
self._logger.trace(`Storage module: Checking ${platform} availability...`);
_.find(self._servicesMonitor.checkList, (value, key) => {
if(key.toLowerCase() === platform.toLowerCase()){
checkFn = value;
return true;
}
});
if (!checkFn || typeof checkFn.fn !== 'function') {
self._logger.trace(`Storage module: Check for ${platform} node does not exist.`);
return callback();
}
checkFn.fn(function (serviceCheckResult) {
if (!serviceCheckResult.status || serviceCheckResult.status === 'off') {
self._logger.trace(`Storage module: ${platform} node not available.`);
return callback('No node');
}
callback();
});
}
_startStorageNode(platform, callback) {
let self = this;
const storageProcessesLauncher = new StorageProcessesLauncher({
logger: self._logger,
events: self._events,
storageConfig: self._storageConfig,
webServerConfig: self._webServerConfig,
blockchainConfig: self._blockchainConfig
});
self._logger.trace(`Storage module: Launching ${platform} process...`);
return storageProcessesLauncher.launchProcess(platform.toLowerCase(), (err) => {
if (err) {
self._logger.error(err);
return callback(err);
}
callback();
});
}
/// Initializes a storage provider for Embark upload
initStorageForEmbark(){
let storageProviderCls = require(`../${this._storageConfig.upload.provider}/index.js`);
let uploadProvider = new storageProviderCls(embark, options); /*eslint no-new: "off"*/
let uploadProvider = new storageProviderCls(this._embark, this._options); /*eslint no-new: "off"*/
if(typeof uploadProvider.commandlineDeploy == 'function') uploadProvider.commandlineDeploy();
if(typeof uploadProvider.setServiceCheck == 'function') uploadProvider.setServiceCheck();
if(typeof uploadProvider.addObjectToConsole == 'function') uploadProvider.addObjectToConsole();
// loop through all available providers and add the provider code to embarkjs
this._storageConfig.available_providers.forEach(providerStr => {
let storageProvider;
// check if we've already instantiated our storage class and reuse
if(providerStr === this._storageConfig.upload.provider){
storageProvider = uploadProvider;
}
// otherwise instantiate the storage provider
else {
let storageProviderCls = require(`../${providerStr}/index.js`);
storageProvider = new storageProviderCls(embark, options); /*eslint no-new: "off"*/
/**
* Initializes a storage provider for EmbarkJS
*
* @return {void}
*/
initStorageForDapp(){
// now we need to add instantiate any dappConnection/available_providers storage providers to add
// their provider code to embarkjs
this._validDappProviders.forEach(dappConn => {
if(!dappConn.provider) return;
let storageProviderCls = require(`../${dappConn.provider}/index.js`);
// override options with dappConnection settings
let storageOptions = this._options;
storageOptions.protocol = dappConn.protocol;
storageOptions.host = dappConn.host;
storageOptions.port = dappConn.port;
// then instantiate the storage provdier class
let storageProvider = new storageProviderCls(this._embark, storageOptions); /*eslint no-new: "off"*/
// register the service check so we can use it to check if the process is running before spawning it
// check that it hasn't already been done above
if(dappConn.provider !== this._storageConfig.upload.provider){
if(typeof storageProvider.setServiceCheck == 'function') storageProvider.setServiceCheck();
}
@ -43,6 +124,11 @@ class Storage {
this.addSetProviders();
}
/**
* Adds the storage provider code (__embarkStorage) to embarkjs
*
* @returns {void}
*/
addProviderToEmbarkJS(){
// TODO: make this a shouldAdd condition
if (this._storageConfig === {} || !this._storageConfig.dappConnection || !this._storageConfig.dappConnection.length) {
@ -54,26 +140,78 @@ class Storage {
this._embark.addCodeToEmbarkJS(code);
}
/**
* Adds the code to call setProviders in embarkjs after embark is ready
*
* @returns {void}
*/
addSetProviders() {
// TODO: make this a shouldAdd condition
if (this._storageConfig === {} || !this._storageConfig.dappConnection || !this._storageConfig.dappConnection.length) {
return;
}
// filter list of dapp connections based on available_providers set in config
let hasSwarm = _.contains(this._storageConfig.available_providers, 'swarm'); // don't need to eval this in every loop iteration
let connectionsToSet = _.filter(this._storageConfig.dappConnection, (conn) => {
return _.contains(this._storageConfig.available_providers, conn.provider) || (conn === '$BZZ' && hasSwarm);
});
let code = `\n__embarkStorage.setProviders(${JSON.stringify(connectionsToSet)});`;
let code = `\n__embarkStorage.setProviders(${JSON.stringify(this._validDappProviders)});`;
let shouldInit = (storageConfig) => {
return (connectionsToSet !== undefined && connectionsToSet.length > 0 && storageConfig.enabled === true);
return (this._validDappProviders !== undefined && this._validDappProviders.length > 0 && storageConfig.enabled === true);
};
this._embark.addProviderInit('storage', code, shouldInit);
}
startStorageProcesses(){
let platform = this._storageConfig.upload.provider;
let self = this;
async.waterfall([
function checkStorageService(callback){
const errorObj = {message: __('Cannot upload: {{platform}} node is not running on {{url}}.', {platform: platform, url: utils.buildUrlFromConfig(self._storageConfig.upload)})};
// start the upload storage node
self._checkStorageEndpoint(platform, function (err) {
if (!err) {
return callback();
}
self._startStorageNode(platform, (err) => {
if (err) {
self.logger.error(err);
return callback(errorObj);
}
// Check endpoint again to see if really did start
self._checkStorageEndpoint(platform, (err) => {
if (err) {
return callback(errorObj);
}
callback();
});
});
});
},
function checkDappConnectionStorageService(callback){
// start any dappConnection storage nodes
self._validDappProviders.forEach(dappConn => {
if(!dappConn.provider || dappConn.provider === platform) return; // don't start the process we've just started above
const errorObj = {message: __('Cannot start {{platform}} node on {{url}}.', {platform: dappConn.provider, url: utils.buildUrlFromConfig(dappConn)})};
self._checkStorageEndpoint(dappConn.provider, function (err) {
if (!err) {
return callback();
}
self._startStorageNode(dappConn.provider, (err) => {
if (err) {
self.logger.error(err);
return callback(errorObj);
}
// Check endpoint again to see if really did start
self._checkStorageEndpoint(dappConn.provider, (err) => {
if (err) {
return callback(errorObj);
}
callback();
});
});
});
});
}
]);
}
}
module.exports = Storage;

9
lib/modules/swarm/index.js
View File

@ -14,12 +14,8 @@ class Swarm {
this.addCheck = options.addCheck;
this.embark = embark;
let host = options.host || options.storageConfig.upload.host;
let port = options.port || options.storageConfig.upload.port;
if(port) port = ':' + port;
else port = '';
let protocol = options.protocol || options.storageConfig.upload.protocol || 'http';
this.providerUrl = `${protocol}://${host}${port}`;
this.providerUrl = utils.buildUrl(options.protocol || options.storageConfig.upload.protocol, options.host || options.storageConfig.upload.host, options.port || options.storageConfig.upload.port);
this.getUrl = options.storageConfig.upload.getUrl || this.providerUrl + '/bzz:/';
this.bzz = new Web3Bzz(this.providerUrl);
@ -29,7 +25,6 @@ class Swarm {
this.upload_swarm = new UploadSwarm({
buildDir: this.buildDir || 'dist/',
storageConfig: this.storageConfig,
connectUrl: this.providerUrl,
getUrl: this.getUrl,
bzz: this.bzz
});

1
lib/modules/swarm/upload.js
View File

@ -7,7 +7,6 @@ class Swarm {
this.buildDir = options.buildDir || 'dist/';
this.bzz = options.bzz;
this.storageConfig = options.storageConfig;
this.providerUrl = options.providerUrl;
this.getUrl = options.getUrl;
}

15
lib/processes/storageProcesses/storageProcessesLauncher.js
View File

@ -12,8 +12,6 @@ class StorageProcessesLauncher {
this.blockchainConfig = options.blockchainConfig;
this.processes = {};
this.cors = this.buildCors();
this.events.on('exit', () => {
Object.keys(this.processes).forEach(processName => {
this.processes[processName].send('exit');
@ -21,14 +19,13 @@ class StorageProcessesLauncher {
});
}
buildCors()
buildCors(storageName)
{
let corsParts = [];
// add our webserver CORS
if(this.webServerConfig.enabled){
if (this.webServerConfig && this.webServerConfig.host) {
let port = this.webServerConfig.port ? `:${this.webServerConfig.port}` : '';
corsParts.push(`${this.webServerConfig.protocol || 'http'}://${this.webServerConfig.host}${port}`);
corsParts.push(utils.buildUrlFromConfig(this.webServerConfig));
}
else corsParts.push('http://localhost:8000');
}
@ -36,6 +33,7 @@ class StorageProcessesLauncher {
// add all dapp connection storage
if(this.storageConfig.enabled) {
this.storageConfig.dappConnection.forEach(dappConn => {
if(dappConn.provider === storageName) return; // do not add CORS URL for ourselves
if(dappConn.getUrl || dappConn.host){
// if getUrl is specified in the config, that needs to be included in cors
@ -46,10 +44,9 @@ class StorageProcessesLauncher {
getUrlParts = getUrlParts.slice(0, 3);
corsParts.push(getUrlParts.join('/'));
}
// use our modified getUrl or in case it wasn't specified, use a built url
// in case getUrl wasn't specified, use a built url
else{
let port = dappConn.port ? `:${dappConn.port}` : '';
corsParts.push(`${dappConn.protocol || 'http'}://${dappConn.host}${port}`);
corsParts.push(utils.buildUrlFromConfig(dappConn));
}
}
});
@ -94,7 +91,7 @@ class StorageProcessesLauncher {
self.processes[storageName].send({
action: constants.blockchain.init, options: {
storageConfig: self.storageConfig,
cors: self.cors
cors: self.buildCors(storageName)
}
});

2
lib/processes/storageProcesses/swarm.js
View File

@ -23,7 +23,7 @@ class SwarmProcess extends ProcessWrapper {
`--password=${fs.dappPath(this.storageConfig.account.password)}`,
`--corsdomain=${self.cors.join(',')}`
];
console.log('SWARM ARGS: ' + JSON.stringify(args));
console.trace('Starting swarm process with arguments: ' + args.join(' '));
this.child = child_process.spawn(this.storageConfig.swarmPath || 'swarm', args);
this.child.on('error', (err) => {

37
lib/utils/utils.js
View File

@ -210,6 +210,39 @@ function normalizeInput(input) {
}).toString() + ']');
}
/**
* Builds a URL
*
* @param {string} protocol
* The URL protocol, defaults to http.
* @param {string} host
* The URL host, required.
* @param {string} port
* The URL port, default to empty string.
* @returns {string} the constructued URL, with defaults
*/
function buildUrl (protocol, host, port){
if(!host) throw new Error('utils.buildUrl: parameter \'host\' is required');
if(port) port = ':' + port;
else port = '';
return `${protocol || 'http'}://${host}${port}`;
}
/**
* Builds a URL
*
* @param {object} configObj Object containing protocol, host, and port to be used to construct the url.
* * protocol {String} (optional) The URL protocol, defaults to http.
* * host {String} (required) The URL host.
* * port {String} (optional) The URL port, default to empty string.
* @returns {string} the constructued URL, with defaults
*/
function buildUrlFromConfig (configObj){
if(!configObj) throw new Error('[utils.buildUrlFromConfig]: config object must cannot be null');
if(!configObj.host) throw new Error('[utils.buildUrlFromConfig]: object must contain a \'host\' property');
return this.buildUrl(configObj.protocol, configObj.host, configObj.port);
}
module.exports = {
joinPath: joinPath,
filesMatchingPattern: filesMatchingPattern,
@ -231,5 +264,7 @@ module.exports = {
getExternalContractUrl,
toChecksumAddress: toChecksumAddress,
sha3: sha3,
normalizeInput
normalizeInput,
buildUrl,
buildUrlFromConfig
};