Handle geth process exit via crash/kill and also via killing `embark blockchain`

First case - run `embark run` which starts a blockchain node, then manually kill the `geth` process. Would throw `{ [Error: connect ECONNREFUSED 127.0.0.1:8543] message: 'connect ECONNREFUSED 127.0.0.1:8543', code: -32603 }` error and ruins the dashboard.

Second case, 1) run `embark blockchain` 2) run `embark run` 3) kill `embark blockchain` throws the error `{ [Error: connect ECONNREFUSED 127.0.0.1:8543] message: 'connect ECONNREFUSED 127.0.0.1:8543', code: -32603 }` and ruins the dashboard.

The first case was solved by having the child blockchain process that spawns geth listen for geth exit, then kill itself.

The second case required updating of `eth-block-tracker` to v4.0.1 inside of the `embark-web3-provider-engine`. v4.0.1 was a major version update and introduced breaking changes. Those changes were handled inside of `embark-web3-provider-engine`, covered in **blocker** PR https://github.com/jrainville/provider-engine/pull/1.

lib/cmds/blockchain/blockchain.js

@@ -17,6 +17,7 @@ var Blockchain = function(options) {
   this.client = options.client;
   this.isDev = options.isDev;
   this.onReadyCallback = options.onReadyCallback || (() => {});
+  this.onExitCallback = options.onExitCallback;
 
   if ((this.blockchainConfig === {} || JSON.stringify(this.blockchainConfig) === '{"enabled":true}') && this.env !== 'development') {
     console.log("===> " + __("warning: running default config on a non-development environment"));
@@ -182,8 +183,24 @@ Blockchain.prototype.run = function() {
       console.log('Geth: ' + data);
     });
     self.child.on('exit', (code) => {
+      let strCode = '';
       if (code) {
-        console.error('Geth exited with error code ' + code);
+        strCode = ' with error code ' + code;
+      } else {
+        strCode = ' with no error code (manually killed?)';
+      }
+      console.error('Geth exited' + strCode);
+
+      if(self.onExitCallback){
+        self.onExitCallback();
+      }
+    });
+
+    self.child.on('uncaughtException', (err) => {
+      console.error('Uncaught geth exception', err);
+
+      if(self.onExitCallback){
+        self.onExitCallback();
       }
     });
   });
@@ -279,10 +296,10 @@ Blockchain.prototype.initChainAndGetAddress = function(callback) {
   });
 };
 
-var BlockchainClient = function(blockchainConfig, client, env, isDev, onReadyCallback) {
+var BlockchainClient = function(blockchainConfig, client, env, isDev, onReadyCallback, onExitCallback) {
   // TODO add other clients at some point
   if (client === 'geth') {
-    return new Blockchain({blockchainConfig, client: GethCommands, env, isDev, onReadyCallback});
+    return new Blockchain({blockchainConfig, client: GethCommands, env, isDev, onReadyCallback, onExitCallback});
   } else {
     throw new Error('unknown client');
   }

lib/cmds/blockchain/blockchainProcess.js

@@ -21,7 +21,8 @@ class BlockchainProcess extends ProcessWrapper {
       this.client,
       this.env,
       this.isDev,
-      this.blockchainReady.bind(this)
+      this.blockchainReady.bind(this),
+      this.blockchainExit.bind(this)
     );
 
     this.blockchain.run();
@@ -31,6 +32,11 @@ class BlockchainProcess extends ProcessWrapper {
     blockchainProcess.send({result: constants.blockchain.blockchainReady});
   }
 
+  blockchainExit() {
+    // tell our parent process that geth has exited
+    blockchainProcess.send({result: constants.blockchain.blockchainExit});
+  }
+
   kill() {
     this.blockchain.kill();
   }

lib/constants.json

@@ -34,6 +34,7 @@
   },
   "blockchain": {
     "blockchainReady": "blockchainReady",
+    "blockchainExit": "blockchainExit",
     "init": "init",
     "initiated": "initiated",
     "servicePortOnProxy": 10

lib/contracts/blockchain.js

@@ -49,7 +49,7 @@ class Blockchain {
     }
 
     const protocol = (this.contractsConfig.deployment.type === "rpc") ? this.contractsConfig.deployment.protocol : 'ws';
-    let provider;
+    
     this.web3Endpoint = utils.buildUrl(protocol, this.contractsConfig.deployment.host, this.contractsConfig.deployment.port);//`${protocol}://${this.contractsConfig.deployment.host}:${this.contractsConfig.deployment.port}`;
     const providerOptions = {
       web3: this.web3,
@@ -60,7 +60,7 @@ class Blockchain {
       type: this.contractsConfig.deployment.type,
       web3Endpoint: self.web3Endpoint
     };
-    provider = new Provider(providerOptions);
+    this.provider = new Provider(providerOptions);
 
     async.waterfall([
       function checkNode(next) {
@@ -78,17 +78,17 @@ class Blockchain {
           self.web3StartedInProcess = true;
           self.startBlockchainNode(() => {
             // Need to re-initialize web3 to connect to the new blockchain node
-            provider.stop();
+            self.provider.stop();
             self.initWeb3(cb);
           });
         });
       },
       function startProvider(next) {
-        provider.startWeb3Provider(next);
+        self.provider.startWeb3Provider(next);
       },
       function fundAccountsIfNeeded(next) {
         self.isWeb3Ready = true;
-        provider.fundAccounts(next);
+        self.provider.fundAccounts(next);
       }
     ], (err) => {
       self.registerWeb3Object();
@@ -121,6 +121,10 @@ class Blockchain {
     self.events.once(constants.blockchain.blockchainReady, () => {
       callback();
     });
+    self.events.once(constants.blockchain.blockchainExit, () => {
+      self.provider.stop();
+      callback();
+    });
   }
 
   registerServiceCheck() {
@@ -157,6 +161,14 @@ class Blockchain {
         if (err && err !== NO_NODE) {
           return cb(err);
         }
+        else if ((statusObj && statusObj.status === 'off') || err === NO_NODE){
+          self.provider.stop();
+          self.events.on('check:backOnline:Ethereum', () => {
+            self.provider.startWeb3Provider(() => {
+              self.logger.trace('web3 provider restarted after ethereum node came back online');
+            });
+          });
+        }
         cb(statusObj);
       });
     }, 5000, 'off');

lib/contracts/provider.js

@@ -40,11 +40,11 @@ class Provider {
     // network connectivity error
     self.engine.on('error', (err) => {
       // report connectivity errors
-      self.logger.error(err.stack);
+      self.logger.error(err);
     });
 
     self.engine.start();
-    self.web3.setProvider(self);
+  self.web3.setProvider(self);
 
     self.accounts = AccountParser.parseAccountsConfig(self.accountsConfig, self.web3, self.logger);
     self.addresses = [];

lib/processes/blockchainProcessLauncher.js

@@ -44,6 +44,14 @@ class BlockchainProcessLauncher {
       this.events.emit(constants.blockchain.blockchainReady);
     });
 
+    this.blockchainProcess.once('result', constants.blockchain.blockchainExit, () => {
+      // telle everyone that our blockchain process (ie geth) died
+      this.events.emit(constants.blockchain.blockchainExit);
+
+      // then kill off the blockchain process
+      this.blockchainProcess.kill();
+    });
+
     this.events.on('exit', () => {
       this.blockchainProcess.send('exit');
     });