status-im
/
embark-area-51
mirror of https://github.com/status-im/embark-area-51.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Embark blockchain logs when running standalone 

When running `embark blockchain` followed by `embark run` previously, logs generated in the standalone `embark blockchain` process were black boxed and not accessible to the main Embark process.

This is fixed by creating a client IPC connection in the `embark blockchain` process that connects to the IPC server connection running in `embark run`. The connection is made by way of polling `ipc.connect` and continues polling even after a connection is made in case `embark run` is killed and restarted without restarting `embark blockchain`.

`LogHandler` was introduced to extrapolate functionality used in `ProcessLauncher` that needed to also be used in the standalone blockchain process. It also caps the number of logs that are stored in memory per process by a constant value defined in `constants.json`.

A `blockchain_listener` was module was created (and run inside of `embark run`) that listens for logs emitted by the `embark blockchain` client IPC and runs them through the `LogHandler`. Additionally, this module registers the API endpoints needed to handle requests for blockchain process logs in the cockpit (which were 404’ing before).

# Conflicts:
#	lib/modules/blockchain_process/blockchain.js
This commit is contained in:
emizzle 2018-10-22 23:56:14 +11:00 committed by Pascal Precht
parent 4b00145e4b
commit 728713a183
No known key found for this signature in database
GPG Key ID: 0EE28D8D6FD85D7D
7 changed files with 248 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
cmd/cmd_controller.js
View File

@ -31,8 +31,8 @@ class EmbarkController {
}
blockchain(env, client) {
this.context = [constants.contexts.blockchain];
return require('../lib/modules/blockchain_process/blockchain.js')(this.config.blockchainConfig, client, env).run();
this.context = [constants.contexts.blockchain];
return require('../lib/modules/blockchain_process/blockchain.js')(this.config.blockchainConfig, client, env, null, null, this.logger, this.events, true).run();
}
simulator(options) {
@ -132,6 +132,7 @@ class EmbarkController {
engine.startService("processManager");
engine.startService("coreProcess");
engine.startService("loggerApi");
engine.startService("blockchainListener");
engine.startService("serviceMonitor");
engine.startService("libraryManager");
engine.startService("codeRunner");
@ -437,73 +438,50 @@ class EmbarkController {
}
scaffold(options) {
this.context = options.context || [constants.contexts.scaffold];
options.onlyCompile = true;
const Engine = require('../lib/core/engine.js');
const engine = new Engine({
env: options.env,
client: options.client,
locale: options.locale,
version: this.version,
embarkConfig: 'embark.json',
interceptLogs: false,
embarkConfig: options.embarkConfig || 'embark.json',
logFile: options.logFile,
logLevel: options.logLevel,
events: options.events,
logger: options.logger,
config: options.config,
plugins: options.plugins,
context: this.context,
webpackConfigName: options.webpackConfigName
context: this.context
});
async.waterfall([
function initEngine(callback) {
function (callback) {
engine.init({}, callback);
},
function startServices(callback) {
engine.startService("scaffolding");
callback();
},
function generateContract(callback) {
engine.events.request('scaffolding:generate:contract', options, function(err, file) {
// Add contract file to the manager
engine.events.request('config:contractsFiles:add', file);
callback();
});
},
function initEngineServices(callback) {
function (callback) {
let pluginList = engine.plugins.listPlugins();
if (pluginList.length > 0) {
engine.logger.info(__("loaded plugins") + ": " + pluginList.join(", "));
}
engine.startService("processManager");
engine.startService("libraryManager");
engine.startService("codeRunner");
engine.startService("web3");
engine.startService("deployment", {onlyCompile: true});
callback();
},
function deploy(callback) {
engine.events.request('deploy:contracts', function(err) {
callback(err);
});
},
function generateUI(callback) {
engine.events.request("scaffolding:generate:ui", options, () => {
callback();
});
engine.startService("processManager");
engine.startService("serviceMonitor");
engine.startService("libraryManager");
engine.startService("pipeline");
engine.startService("deployment", {onlyCompile: true});
engine.startService("web3");
engine.startService("scaffolding");
engine.events.request('deploy:contracts', callback);
}
], function(err) {
], (err) => {
if (err) {
engine.logger.error(__("Error generating the UI: "));
engine.logger.error(err.message || err);
process.exit(1);
engine.logger.error(err.message);
engine.logger.info(err.stack);
} else {
engine.events.request("scaffolding:generate", options, () => {
engine.logger.info(__("finished generating the UI").underline);
process.exit();
});
}
engine.logger.info(__("finished generating the UI").underline);
engine.logger.info(__("To see the result, execute {{cmd}} and go to /{{contract}}.html", {cmd: 'embark run'.underline, contract: options.contract}));
process.exit();
});
}

3
lib/constants.json
View File

@ -54,6 +54,7 @@
"gasLimit": 6000000
},
"logs": {
"logPath": ".embark/logs/"
"logPath": ".embark/logs/",
"maxLogLength": 1500
}
}

9
lib/core/engine.js
View File

@ -78,7 +78,8 @@ class Engine {
"codeCoverage": this.codeCoverageService,
"scaffolding": this.scaffoldingService,
"coreProcess": this.coreProcessService,
"loggerApi": this.loggerApiService
"loggerApi": this.loggerApiService,
"blockchainListener": this.blockchainListenerService
};
let service = services[serviceName];
@ -92,6 +93,12 @@ class Engine {
return service.apply(this, [options]);
}
blockchainListenerService(_options){
this.registerModule('blockchain_listener', {
ipc: this.ipc
});
}
coreProcessService(_options){
this.registerModule('core_process', {
events: this.events

42
lib/core/processes/processLauncher.js
View File

@ -2,6 +2,7 @@ const child_process = require('child_process');
const constants = require('../../constants');
const path = require('path');
const utils = require('../../utils/utils');
const LogHandler = require('../../utils/logHandler');
let processCount = 1;
class ProcessLauncher {
@ -30,6 +31,7 @@ class ProcessLauncher {
this.exitCallback = options.exitCallback;
this.embark = options.embark;
this.logs = [];
this.logHandler = new LogHandler({events: this.events, logger: this.logger, processName: this.name, silent: this.silent});
this.subscriptions = {};
this._subscribeToMessages();
@ -51,7 +53,7 @@ class ProcessLauncher {
self.logger.error(msg.error);
}
if (msg.result === constants.process.log) {
return self._handleLog(msg);
return self.logHandler.handleLog(msg);
}
if (msg.event) {
return self._handleEvent(msg);
@ -93,44 +95,6 @@ class ProcessLauncher {
);
}
// Translates logs from the child process to the logger
_handleLog(msg) {
// Sometimes messages come in with line breaks, so we need to break them up accordingly.
let processedMessages = [];
// Ensure that `msg.message` is an array, so we process this consistently. Sometimes it
// is an Array, sometimes it is a string.
if(typeof msg.message === 'string') {
processedMessages = [msg.message];
} else {
msg.message.forEach(message => {
let lines = message.split("\n");
lines.forEach(line => processedMessages.push(line));
});
}
const timestamp = new Date().getTime();
processedMessages.forEach((message) => {
const log = {
msg: message,
msg_clear: message.stripColors,
logLevel: msg.logLevel,
name: this.name,
timestamp
};
const id = this.logs.push(log) - 1;
this.events.emit(`process-log-${this.name}`, id, log);
if (this.silent && msg.type !== 'error') {
return;
}
if (this.logger[msg.type]) {
return this.logger[msg.type](utils.normalizeInput(message));
}
this.logger.debug(utils.normalizeInput(message));
});
}
// Handle event calls from the child process
_handleEvent(msg) {
const self = this;

54
lib/modules/blockchain_listener/index.js Normal file
View File

@ -0,0 +1,54 @@
const LogHandler = require('../../utils/logHandler');
const PROCESS_NAME = 'blockchain';
class BlockchainListener {
constructor(embark, options) {
this.embark = embark;
this.events = embark.events;
this.logger = embark.logger;
this.ipc = options.ipc;
this.logHandler = new LogHandler({events: this.events, logger: this.logger, processName: PROCESS_NAME, silent: true});
if (this.ipc.isServer()) {
this._listenToLogs();
}
this._registerApi();
}
_listenToLogs() {
this.ipc.on('blockchain:log', ({logLevel, message}) => {
this.logHandler.handleLog({logLevel, message});
});
}
_registerApi() {
// This route is overriden by `processLauncher` when the blockchain
// process is launched (ie when not in blockchain standalone mode).
// The order is determined by the order in which the engine starts
// it's services, with `blockchain_process` coming before `web3`.
const apiRoute = '/embark-api/process-logs/' + PROCESS_NAME;
this.embark.registerAPICall(
'ws',
apiRoute,
(ws, _req) => {
this.events.on('process-log-' + PROCESS_NAME, function (id, log) {
ws.send(JSON.stringify(Object.assign(log, {id})), () => {});
});
}
);
this.embark.registerAPICall(
'get',
apiRoute,
(req, res) => {
let limit = parseInt(req.query.limit, 10);
if (!Number.isInteger(limit)) limit = 0;
const result = this.logHandler.logs.map((log, id) => Object.assign(log, {id})).slice(limit);
res.send(JSON.stringify(result));
}
);
}
}
module.exports = BlockchainListener;

129
lib/modules/blockchain_process/blockchain.js
View File

@ -1,27 +1,31 @@
const async = require('async');
const {spawn, exec} = require('child_process');
const fs = require('../../core/fs.js');
const constants = require('../../constants.json');
const utils = require('../../utils/utils.js');
const GethClient = require('./gethClient.js');
const ParityClient = require('./parityClient.js');
const DevFunds = require('./dev_funds.js');
const proxy = require('./proxy');
const Ipc = require('../../core/ipc');
const {defaultHost, dockerHostSwap} = require('../../utils/host');
const Logger = require('../../core/logger');
/*eslint complexity: ["error", 36]*/
// time between IPC connection attmpts (in ms)
const IPC_CONNECT_INTERVAL = 2000;
/*eslint complexity: ["error", 38]*/
var Blockchain = function(userConfig, clientClass) {
this.userConfig = userConfig;
this.env = userConfig.env || 'development';
this.isDev = userConfig.isDev;
this.onReadyCallback = userConfig.onReadyCallback || (() => {});
this.onExitCallback = userConfig.onExitCallback;
this.logger = userConfig.logger || new Logger({logLevel: 'debug'});
this.events = userConfig.events;
this.proxyIpc = null;
this.isStandalone = userConfig.isStandalone;
let defaultWsApi = clientClass.DEFAULTS.WS_API;
if (this.isDev) defaultWsApi = clientClass.DEFAULTS.DEV_WS_API;
@ -75,29 +79,62 @@ var Blockchain = function(userConfig, clientClass) {
const spaceMessage = 'The path for %s in blockchain config contains spaces, please remove them';
if (this.config.datadir && this.config.datadir.indexOf(' ') > 0) {
console.error(__(spaceMessage, 'datadir'));
this.logger.error(__(spaceMessage, 'datadir'));
process.exit();
}
if (this.config.account.password && this.config.account.password.indexOf(' ') > 0) {
console.error(__(spaceMessage, 'account.password'));
this.logger.error(__(spaceMessage, 'account.password'));
process.exit();
}
if (this.config.genesisBlock && this.config.genesisBlock.indexOf(' ') > 0) {
console.error(__(spaceMessage, 'genesisBlock'));
this.logger.error(__(spaceMessage, 'genesisBlock'));
process.exit();
}
this.initProxy();
this.client = new clientClass({config: this.config, env: this.env, isDev: this.isDev});
this.initStandaloneProcess();
};
Blockchain.prototype.initProxy = function() {
/**
* Polls for a connection to an IPC server (generally this is set up
* in the Embark process). Once connected, any logs logged to the
* Logger will be shipped off to the IPC server. In the case of `embark
* run`, the BlockchainListener module is listening for these logs.
*
* @returns {void}
*/
Blockchain.prototype.initStandaloneProcess = function () {
if (this.isStandalone) {
// on every log logged in logger (say that 3x fast), send the log
// to the IPC serve listening (only if we're connected of course)
this.events.on('log', (logLevel, message) => {
if (this.ipc.connected) {
this.ipc.request('blockchain:log', {logLevel, message});
}
});
this.ipc = new Ipc({ipcRole: 'client'});
// Wait for an IPC server to start (ie `embark run`) by polling `.connect()`.
// Do not kill this interval as the IPC server may restart (ie restart
// `embark run` without restarting `embark blockchain`)
setInterval(() => {
if (!this.ipc.connected) {
this.ipc.connect(() => {});
}
}, IPC_CONNECT_INTERVAL);
}
};
Blockchain.prototype.initProxy = function () {
if (this.config.proxy) {
this.config.rpcPort += constants.blockchain.servicePortOnProxy;
this.config.wsPort += constants.blockchain.servicePortOnProxy;
}
};
Blockchain.prototype.setupProxy = async function() {
Blockchain.prototype.setupProxy = async function () {
if (!this.proxyIpc) this.proxyIpc = new Ipc({ipcRole: 'client'});
let wsProxy;
@ -108,7 +145,7 @@ Blockchain.prototype.setupProxy = async function() {
[this.rpcProxy, this.wsProxy] = await Promise.all([proxy.serve(this.proxyIpc, this.config.rpcHost, this.config.rpcPort, false), wsProxy]);
};
Blockchain.prototype.shutdownProxy = function() {
Blockchain.prototype.shutdownProxy = function () {
if (!this.config.proxy) {
return;
}
@ -117,21 +154,21 @@ Blockchain.prototype.shutdownProxy = function() {
if (this.wsProxy) this.wsProxy.close();
};
Blockchain.prototype.runCommand = function(cmd, options, callback) {
console.log(__("running: %s", cmd.underline).green);
Blockchain.prototype.runCommand = function (cmd, options, callback) {
this.logger.info(__("running: %s", cmd.underline).green);
if (this.config.silent) {
options.silent = true;
}
return exec(cmd, options, callback);
};
Blockchain.prototype.run = function() {
Blockchain.prototype.run = function () {
var self = this;
console.log("===============================================================================".magenta);
console.log("===============================================================================".magenta);
console.log(__("Embark Blockchain using %s", self.client.prettyName.underline).magenta);
console.log("===============================================================================".magenta);
console.log("===============================================================================".magenta);
this.logger.info("===============================================================================".magenta);
this.logger.info("===============================================================================".magenta);
this.logger.info(__("Embark Blockchain using %s", self.client.prettyName.underline).magenta);
this.logger.info("===============================================================================".magenta);
this.logger.info("===============================================================================".magenta);
if (self.client.name === 'geth') this.checkPathLength();
@ -157,35 +194,35 @@ Blockchain.prototype.run = function() {
});
},
function getMainCommand(next) {
self.client.mainCommand(address, function(cmd, args) {
self.client.mainCommand(address, function (cmd, args) {
next(null, cmd, args);
}, true);
}
], function(err, cmd, args) {
if (err) {
console.error(err.message);
self.logger.error(err.message);
return;
}
args = utils.compact(args);
let full_cmd = cmd + " " + args.join(' ');
console.log(__("running: %s", full_cmd.underline).green);
self.logger.info(__("running: %s", full_cmd.underline).green);
self.child = spawn(cmd, args, {cwd: process.cwd()});
self.child.on('error', (err) => {
err = err.toString();
console.error('Blockchain error: ', err);
self.logger.error('Blockchain error: ', err);
if (self.env === 'development' && err.indexOf('Failed to unlock') > 0) {
console.error('\n' + __('Development blockchain has changed to use the --dev option.').yellow);
console.error(__('You can reset your workspace to fix the problem with').yellow + ' embark reset'.cyan);
console.error(__('Otherwise, you can change your data directory in blockchain.json (datadir)').yellow);
self.logger.error('\n' + __('Development blockchain has changed to use the --dev option.').yellow);
self.logger.error(__('You can reset your workspace to fix the problem with').yellow + ' embark reset'.cyan);
self.logger.error(__('Otherwise, you can change your data directory in blockchain.json (datadir)').yellow);
}
});
// TOCHECK I don't understand why stderr and stdout are reverted.
// This happens with Geth and Parity, so it does not seems a client problem
self.child.stdout.on('data', (data) => {
console.error(`${self.client.name} error: ${data}`);
this.logger.info(`${self.client.name} error: ${data}`);
});
self.child.stderr.on('data', async (data) => {
@ -194,7 +231,7 @@ Blockchain.prototype.run = function() {
self.readyCalled = true;
if (self.isDev) {
self.fundAccounts((err) => {
if (err) console.error('Error funding accounts', err);
if (err) this.logger.error('Error funding accounts', err);
});
}
if (self.config.proxy) {
@ -202,7 +239,7 @@ Blockchain.prototype.run = function() {
}
self.readyCallback();
}
console.log(`${self.client.name}: ${data}`);
this.logger.info(`${self.client.name}: ${data}`);
});
self.child.on('exit', (code) => {
@ -212,14 +249,14 @@ Blockchain.prototype.run = function() {
} else {
strCode = 'with no error code (manually killed?)';
}
console.error(self.client.name + ' exited ' + strCode);
self.logger.error(self.client.name + ' exited ' + strCode);
if (self.onExitCallback) {
self.onExitCallback();
}
});
self.child.on('uncaughtException', (err) => {
console.error('Uncaught ' + self.client.name + ' exception', err);
self.logger.error('Uncaught ' + self.client.name + ' exception', err);
if (self.onExitCallback) {
self.onExitCallback();
}
@ -235,7 +272,7 @@ Blockchain.prototype.fundAccounts = function(cb) {
});
};
Blockchain.prototype.readyCallback = function() {
Blockchain.prototype.readyCallback = function () {
if (this.onReadyCallback) {
this.onReadyCallback();
}
@ -244,25 +281,25 @@ Blockchain.prototype.readyCallback = function() {
}
};
Blockchain.prototype.kill = function() {
Blockchain.prototype.kill = function () {
this.shutdownProxy();
if (this.child) {
this.child.kill();
}
};
Blockchain.prototype.checkPathLength = function() {
Blockchain.prototype.checkPathLength = function () {
let dappPath = fs.dappPath('');
if (dappPath.length > 66) {
// console.error is captured and sent to the console output regardless of silent setting
console.error("===============================================================================".yellow);
console.error("===========> ".yellow + __('WARNING! ÐApp path length is too long: ').yellow + dappPath.yellow);
console.error("===========> ".yellow + __('This is known to cause issues with starting geth, please consider reducing your ÐApp path\'s length to 66 characters or less.').yellow);
console.error("===============================================================================".yellow);
// this.logger.error is captured and sent to the console output regardless of silent setting
this.logger.error("===============================================================================".yellow);
this.logger.error("===========> ".yellow + __('WARNING! ÐApp path length is too long: ').yellow + dappPath.yellow);
this.logger.error("===========> ".yellow + __('This is known to cause issues with starting geth, please consider reducing your ÐApp path\'s length to 66 characters or less.').yellow);
this.logger.error("===============================================================================".yellow);
}
};
Blockchain.prototype.isClientInstalled = function(callback) {
Blockchain.prototype.isClientInstalled = function (callback) {
let versionCmd = this.client.determineVersionCommand();
this.runCommand(versionCmd, {}, (err, stdout, stderr) => {
if (err || !stdout || stderr.indexOf("not found") >= 0 || stdout.indexOf("not found") >= 0) {
@ -341,7 +378,7 @@ Blockchain.prototype.initDevChain = function(callback) {
});
};
Blockchain.prototype.initChainAndGetAddress = function(callback) {
Blockchain.prototype.initChainAndGetAddress = function (callback) {
const self = this;
let address = null;
const ALREADY_INITIALIZED = 'already';
@ -358,7 +395,7 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) {
function listAccounts(next) {
self.runCommand(self.client.listAccountsCommand(), {}, (err, stdout, _stderr) => {
if (err || stdout === undefined || stdout.indexOf("Fatal") >= 0) {
console.log(__("no accounts found").green);
this.logger.info(__("no accounts found").green);
return next();
}
let firstAccountFound = self.client.parseListAccountsCommandResultToAddress(stdout);
@ -366,7 +403,7 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) {
console.log(__("no accounts found").green);
return next();
}
console.log(__("already initialized").green);
this.logger.info(__("already initialized").green);
address = firstAccountFound;
next(ALREADY_INITIALIZED);
});
@ -376,7 +413,7 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) {
if (!self.config.genesisBlock || self.client.name === 'parity') {
return next();
}
console.log(__("initializing genesis block").green);
this.logger.info(__("initializing genesis block").green);
self.runCommand(self.client.initGenesisCommmand(), {}, (err, _stdout, _stderr) => {
next(err);
});
@ -398,9 +435,9 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) {
});
};
var BlockchainClient = function(userConfig, clientName, env, onReadyCallback, onExitCallback) {
var BlockchainClient = function(userConfig, clientName, env, onReadyCallback, onExitCallback, logger, events, isStandalone) {
if ((userConfig === {} || JSON.stringify(userConfig) === '{"enabled":true}') && env !== 'development') {
console.log("===> " + __("warning: running default config on a non-development environment"));
logger.info("===> " + __("warning: running default config on a non-development environment"));
}
// if client is not set in preferences, default is geth
if (!userConfig.ethereumClientName) userConfig.ethereumClientName = 'geth';
@ -416,7 +453,7 @@ var BlockchainClient = function(userConfig, clientName, env, onReadyCallback, on
case 'parity':
clientClass = ParityClient;
break;
return new Blockchain({blockchainConfig, client: GethCommands, env, isDev, onReadyCallback, onExitCallback, logger, events, isStandalone});
default:
console.error(__('Unknow client "%s". Please use one of the following: %s', userConfig.ethereumClientName, 'geth, parity'));
process.exit();

71
lib/utils/logHandler.js Normal file
View File

@ -0,0 +1,71 @@
const utils = require('./utils');
// define max number of logs to keep in memory for this process
// to prevent runaway memory leak
const MAX_LOGS = require('../constants').logs.maxLogLength;
class LogHandler {
constructor({events, logger, processName, silent}) {
this.events = events;
this.logger = logger;
this.processName = processName;
this.silent = silent;
this.logs = [];
this.removedCount = 0;
}
/**
* Servers as an interception of logs, normalises the message output and
* adds metadata (timestamp, id) the data,
* stores the log in memory, then sends it to the logger for output. Max
* number of logs stored in memory is capped by MAX_LOGS
* @param {Object} msg Object containing the log message (msg.message)
*
* @returns {void}
*/
handleLog(msg) {
if(!msg) return;
// Sometimes messages come in with line breaks, so we need to break them up accordingly.
let processedMessages = [];
// Ensure that `msg.message` is an array, so we process this consistently. Sometimes it
// is an Array, sometimes it is a string.
if(typeof msg.message === 'string') {
processedMessages = [msg.message];
} else {
msg.message.forEach(message => {
let lines = message.split("\n");
lines.forEach(line => processedMessages.push(line));
});
}
const timestamp = new Date().getTime();
processedMessages.forEach((message) => {
const log = {
msg: message,
msg_clear: message.stripColors,
logLevel: msg.logLevel,
name: this.processName,
timestamp
};
if(this.logs.length >= MAX_LOGS){
this.logs.shift();
this.removedCount++;
}
const id = this.logs.push(log) - 1 + this.removedCount;
this.events.emit(`process-log-${this.processName}`, id, log);
if (this.silent && msg.type !== 'error') {
return;
}
if (this.logger[msg.type]) {
return this.logger[msg.type](utils.normalizeInput(message));
}
this.logger.debug(utils.normalizeInput(message));
});
}
}
module.exports = LogHandler;