From 4ce7a179fd6d432d309e5e6a26dd05c714c909b0 Mon Sep 17 00:00:00 2001 From: emizzle Date: Fri, 22 Jun 2018 22:52:15 +1000 Subject: [PATCH 1/3] Handle geth process exit via crash/kill and also via killing `embark blockchain` First case - run `embark run` which starts a blockchain node, then manually kill the `geth` process. Would throw `{ [Error: connect ECONNREFUSED 127.0.0.1:8543] message: 'connect ECONNREFUSED 127.0.0.1:8543', code: -32603 }` error and ruins the dashboard. Second case, 1) run `embark blockchain` 2) run `embark run` 3) kill `embark blockchain` throws the error `{ [Error: connect ECONNREFUSED 127.0.0.1:8543] message: 'connect ECONNREFUSED 127.0.0.1:8543', code: -32603 }` and ruins the dashboard. The first case was solved by having the child blockchain process that spawns geth listen for geth exit, then kill itself. The second case required updating of `eth-block-tracker` to v4.0.1 inside of the `embark-web3-provider-engine`. v4.0.1 was a major version update and introduced breaking changes. Those changes were handled inside of `embark-web3-provider-engine`, covered in **blocker** PR https://github.com/jrainville/provider-engine/pull/1. --- lib/cmds/blockchain/blockchain.js | 23 +++++++++++++++++++--- lib/cmds/blockchain/blockchainProcess.js | 8 +++++++- lib/constants.json | 1 + lib/contracts/blockchain.js | 22 ++++++++++++++++----- lib/contracts/provider.js | 4 ++-- lib/processes/blockchainProcessLauncher.js | 8 ++++++++ 6 files changed, 55 insertions(+), 11 deletions(-) diff --git a/lib/cmds/blockchain/blockchain.js b/lib/cmds/blockchain/blockchain.js index f2e9cf99b..2ab294b10 100644 --- a/lib/cmds/blockchain/blockchain.js +++ b/lib/cmds/blockchain/blockchain.js @@ -14,6 +14,7 @@ var Blockchain = function(options) { this.client = options.client; this.isDev = options.isDev; this.onReadyCallback = options.onReadyCallback; + this.onExitCallback = options.onExitCallback; if ((this.blockchainConfig === {} || JSON.stringify(this.blockchainConfig) === '{"enabled":true}') && this.env !== 'development') { console.log("===> " + __("warning: running default config on a non-development environment")); @@ -167,8 +168,24 @@ Blockchain.prototype.run = function() { console.log('Geth: ' + data); }); self.child.on('exit', (code) => { + let strCode = ''; if (code) { - console.error('Geth exited with error code ' + code); + strCode = ' with error code ' + code; + } else { + strCode = ' with no error code (manually killed?)'; + } + console.error('Geth exited' + strCode); + + if(self.onExitCallback){ + self.onExitCallback(); + } + }); + + self.child.on('uncaughtException', (err) => { + console.error('Uncaught geth exception', err); + + if(self.onExitCallback){ + self.onExitCallback(); } }); }); @@ -252,10 +269,10 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) { }); }; -var BlockchainClient = function(blockchainConfig, client, env, isDev, onReadyCallback) { +var BlockchainClient = function(blockchainConfig, client, env, isDev, onReadyCallback, onExitCallback) { // TODO add other clients at some point if (client === 'geth') { - return new Blockchain({blockchainConfig, client: GethCommands, env, isDev, onReadyCallback}); + return new Blockchain({blockchainConfig, client: GethCommands, env, isDev, onReadyCallback, onExitCallback}); } else { throw new Error('unknown client'); } diff --git a/lib/cmds/blockchain/blockchainProcess.js b/lib/cmds/blockchain/blockchainProcess.js index e9d3724fd..013174210 100644 --- a/lib/cmds/blockchain/blockchainProcess.js +++ b/lib/cmds/blockchain/blockchainProcess.js @@ -21,7 +21,8 @@ class BlockchainProcess extends ProcessWrapper { this.client, this.env, this.isDev, - this.blockchainReady.bind(this) + this.blockchainReady.bind(this), + this.blockchainExit.bind(this) ); this.blockchain.run(); @@ -31,6 +32,11 @@ class BlockchainProcess extends ProcessWrapper { blockchainProcess.send({result: constants.blockchain.blockchainReady}); } + blockchainExit() { + // tell our parent process that geth has exited + blockchainProcess.send({result: constants.blockchain.blockchainExit}); + } + kill() { this.blockchain.kill(); } diff --git a/lib/constants.json b/lib/constants.json index d8a0759be..caf92da9c 100644 --- a/lib/constants.json +++ b/lib/constants.json @@ -34,6 +34,7 @@ }, "blockchain": { "blockchainReady": "blockchainReady", + "blockchainExit": "blockchainExit", "init": "init", "initiated": "initiated", "servicePortOnProxy": 10 diff --git a/lib/contracts/blockchain.js b/lib/contracts/blockchain.js index 6ae0c9c76..059f014a2 100644 --- a/lib/contracts/blockchain.js +++ b/lib/contracts/blockchain.js @@ -48,7 +48,7 @@ class Blockchain { } const protocol = (this.contractsConfig.deployment.type === "rpc") ? this.contractsConfig.deployment.protocol : 'ws'; - let provider; + this.web3Endpoint = utils.buildUrl(protocol, this.contractsConfig.deployment.host, this.contractsConfig.deployment.port);//`${protocol}://${this.contractsConfig.deployment.host}:${this.contractsConfig.deployment.port}`; const providerOptions = { @@ -60,7 +60,7 @@ class Blockchain { type: this.contractsConfig.deployment.type, web3Endpoint: self.web3Endpoint }; - provider = new Provider(providerOptions); + this.provider = new Provider(providerOptions); async.waterfall([ function checkNode(next) { @@ -78,16 +78,16 @@ class Blockchain { self.web3StartedInProcess = true; self.startBlockchainNode(() => { // Need to re-initialize web3 to connect to the new blockchain node - provider.stop(); + self.provider.stop(); self.initWeb3(cb); }); }); }, function startProvider(next) { - provider.startWeb3Provider(next); + self.provider.startWeb3Provider(next); }, function fundAccountsIfNeeded(next) { - provider.fundAccounts(next); + self.provider.fundAccounts(next); } ], (err) => { self.registerWeb3Object(); @@ -120,6 +120,10 @@ class Blockchain { self.events.once(constants.blockchain.blockchainReady, () => { callback(); }); + self.events.once(constants.blockchain.blockchainExit, () => { + self.provider.stop(); + callback(); + }); } registerServiceCheck() { @@ -156,6 +160,14 @@ class Blockchain { if (err && err !== NO_NODE) { return cb(err); } + else if ((statusObj && statusObj.status === 'off') || err === NO_NODE){ + self.provider.stop(); + self.events.on('check:backOnline:Ethereum', () => { + self.provider.startWeb3Provider(() => { + self.logger.trace('web3 provider restarted after ethereum node came back online'); + }); + }); + } cb(statusObj); }); }, 5000, 'off'); diff --git a/lib/contracts/provider.js b/lib/contracts/provider.js index 59e128fbb..7d9b088d4 100644 --- a/lib/contracts/provider.js +++ b/lib/contracts/provider.js @@ -40,11 +40,11 @@ class Provider { // network connectivity error self.engine.on('error', (err) => { // report connectivity errors - self.logger.error(err.stack); + self.logger.error(err); }); self.engine.start(); - self.web3.setProvider(self); + self.web3.setProvider(self); self.accounts = AccountParser.parseAccountsConfig(self.accountsConfig, self.web3, self.logger); self.addresses = []; diff --git a/lib/processes/blockchainProcessLauncher.js b/lib/processes/blockchainProcessLauncher.js index ba30dbd5f..68c54eb87 100644 --- a/lib/processes/blockchainProcessLauncher.js +++ b/lib/processes/blockchainProcessLauncher.js @@ -44,6 +44,14 @@ class BlockchainProcessLauncher { this.events.emit(constants.blockchain.blockchainReady); }); + this.blockchainProcess.once('result', constants.blockchain.blockchainExit, () => { + // telle everyone that our blockchain process (ie geth) died + this.events.emit(constants.blockchain.blockchainExit); + + // then kill off the blockchain process + this.blockchainProcess.kill(); + }); + this.events.on('exit', () => { this.blockchainProcess.send('exit'); }); From fe22a40146606d29a4eb291bfc5a3283801d9113 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Fri, 22 Jun 2018 10:07:02 -0400 Subject: [PATCH 2/3] update provider version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e53e04bc4..c5322a11b 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "uuid": "^3.2.1", "viz.js": "^1.8.1", "web3": "1.0.0-beta.34", - "embark-web3-provider-engine": "14.0.6", + "embark-web3-provider-engine": "14.0.7", "webpack": "^3.10.0", "window-size": "^1.1.0" }, From f67a807ac8693e4a51bd1efce439f252170e6585 Mon Sep 17 00:00:00 2001 From: emizzle Date: Tue, 26 Jun 2018 12:59:14 +1000 Subject: [PATCH 3/3] Updated online/offline event checks so they are run during initialisation. Changed the online event to `once` and set it to be bound every time the node goes offline. The above changes handle the case where: 1) `embark run` runs and starts geth. 2) geth is killed manually 3) `embark blockchain` is run in separate process to restart geth 4) the `embark run` process detects this change and restarts the web3 provider and recompiles/deploys/builds Every time `embark blochain` is restarted, an error is appended and all are emitted from the `eth-block-tracker`. This is a bug but can't figure out where it originates. The downside is that if, for example, `embark blockchain` is restarted 4 times, there will be 4 errors emitted from the `eth-block-tracker`. Because of this, errors emitted from `eth-block-tracker` have been reduced to trace to avoid clogging the logs. --- lib/cmds/blockchain/blockchain.js | 2 +- lib/contracts/blockchain.js | 22 ++++++++++++++-------- lib/contracts/provider.js | 8 ++++++-- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/cmds/blockchain/blockchain.js b/lib/cmds/blockchain/blockchain.js index 2ab294b10..2689ac477 100644 --- a/lib/cmds/blockchain/blockchain.js +++ b/lib/cmds/blockchain/blockchain.js @@ -168,7 +168,7 @@ Blockchain.prototype.run = function() { console.log('Geth: ' + data); }); self.child.on('exit', (code) => { - let strCode = ''; + let strCode; if (code) { strCode = ' with error code ' + code; } else { diff --git a/lib/contracts/blockchain.js b/lib/contracts/blockchain.js index 059f014a2..38cfe9276 100644 --- a/lib/contracts/blockchain.js +++ b/lib/contracts/blockchain.js @@ -73,6 +73,20 @@ class Blockchain { if (!err) { self.isWeb3Ready = true; self.events.emit(WEB3_READY); + // if the ethereum node goes offline, we need a check to ensure + // the provider is also stopped + self.events.on('check:wentOffline:Ethereum', () => { + self.logger.trace('Ethereum went offline: stopping web3 provider...'); + self.provider.stop(); + + // once the node goes back online, we can restart the provider + self.events.once('check:backOnline:Ethereum', () => { + self.logger.trace('Ethereum back online: starting web3 provider...'); + self.provider.startWeb3Provider(() => { + self.logger.trace('web3 provider restarted after ethereum node came back online'); + }); + }); + }); return next(); } self.web3StartedInProcess = true; @@ -160,14 +174,6 @@ class Blockchain { if (err && err !== NO_NODE) { return cb(err); } - else if ((statusObj && statusObj.status === 'off') || err === NO_NODE){ - self.provider.stop(); - self.events.on('check:backOnline:Ethereum', () => { - self.provider.startWeb3Provider(() => { - self.logger.trace('web3 provider restarted after ethereum node came back online'); - }); - }); - } cb(statusObj); }); }, 5000, 'off'); diff --git a/lib/contracts/provider.js b/lib/contracts/provider.js index 7d9b088d4..e7c5ce914 100644 --- a/lib/contracts/provider.js +++ b/lib/contracts/provider.js @@ -39,8 +39,12 @@ class Provider { // network connectivity error self.engine.on('error', (err) => { - // report connectivity errors - self.logger.error(err); + // report connectivity errors as trace due to polling + self.logger.trace('web3 provider error: ', err); + self.logger.trace('stopping web3 provider due to error'); + + // prevent continuous polling errors + self.engine.stop(); }); self.engine.start();