fix(@embark/console): Fix console not working with VM2/monorepo

The console was not working correctly with the latest VM2/monorepo updates. This PR addresses namely fixes this problem, but also adds a few more notable changes:

* SIGNIFICANT improvement in loading time for `embark console` with an already running `embark run`. This is due to removing unneeded services starting, and instead forwarding user input to the main `embark run` process.
* All user input commands are now forwarded to the `embark run` process via IPC insteaad of evaluating the command in the `embark console` process.
* Removed IPC console history as it's no longer needed due to the above. Side effects:
  ** The signature of the `runcode:eval` and `runcode:register` events was changed to remove the `toRecord` parameter.
  ** Old `runcode:eval` signature: `events.request("runcode:eval", "code to be evaluated", (err, result) => {}, isNotUserInput, tolerateError)`
  ** New `runcode:eval` signature: `events.request("runcode:eval", "code to be evaluated", (err, result) => {}, tolerateError)`
  ** Old `runcode:register` signature: `events.request("runcode:register", "varName", variableValue, toRecord, (err, result) => {})`
  ** New `runcode:register` signature: `events.request("runcode:register", "varName", variableValue, (err, result) => {})`

* Removed unneeded `forceRegister` flag.
* Removed the `VM.getWeb3Config` method as it's no longer being used (EmbarkJS contracts are pulled out from the VM instead).
* Updated `web3Connector` `blockchain:connector:ready` to allow for event requests or event emissions.
* In the tests, removed the initial `initWeb3Provider` in the `init` as it was being called twice.
* In the tests, removed the `web3Connector` check message as the tests are now using the Console, and the console does this check. This was causing duplicate messages to be displayed in the output.
* Fix `web3 is not defined` browser error

packages/embark/src/cmd/cmd_controller.js

@@ -278,6 +278,8 @@ class EmbarkController {
       webpackConfigName: options.webpackConfigName
     });
 
+    const isSecondaryProcess = (engine) => { return engine.ipc.connected && engine.ipc.isClient(); };
+
     async.waterfall([
       function initEngine(callback) {
         engine.init({}, callback);
@@ -288,63 +290,36 @@ class EmbarkController {
           engine.logger.info(__("loaded plugins") + ": " + pluginList.join(", "));
         }
 
-        if (engine.ipc.connected) {
+        engine.startService("web3");
-          engine.startService("codeRunner");
+        engine.startService("deployment");
-          engine.startService("console");
+        engine.startService("codeGenerator");
+        engine.startService("codeRunner");
+        engine.startService("console");
+        
+        if (isSecondaryProcess(engine)) {
           return callback();
         }
         engine.startService("processManager");
         engine.startService("serviceMonitor");
         engine.startService("libraryManager");
-        engine.startService("codeRunner");
-        engine.startService("web3");
         engine.startService("pipeline");
-        engine.startService("deployment");
         engine.startService("storage");
-        engine.startService("codeGenerator");
-        engine.startService("console");
         engine.startService("cockpit");
         engine.startService("pluginCommand");
-        engine.events.once('check:backOnline:Ethereum', () => callback());
+        engine.events.request('blockchain:ready', callback);
       },
       function ipcConnect(callback) {
         // Do specific work in case we are connected to a socket:
         //  - Setup Web3
         //  - Apply history
-        if(!engine.ipc.connected || engine.ipc.isServer()) {
+        if(isSecondaryProcess(engine)) {
           return callback();
         }
-        const Provider = require('../lib/modules/blockchain_connector/provider');
+        engine.events.request("console:provider:ready", callback);
-        const Web3 = require('web3');
-        let web3 = new Web3();
-        engine.ipc.request("runcode:getCommands", null, (_, {web3Config, commands}) => {
-          const providerOptions = {
-            web3: web3,
-            accountsConfig: engine.config.contractsConfig.deployment.accounts,
-            blockchainConfig: engine.config.blockchainConfig,
-            logger: engine.logger,
-            isDev: engine.isDev,
-            type: engine.config.contractsConfig.deployment.type,
-            web3Endpoint: web3Config.providerUrl
-          };
-          const provider = new Provider(providerOptions);
-          web3.eth.defaultAccount = web3Config.defaultAccount;
-          provider.startWeb3Provider(() => {
-            engine.events.emit("runcode:register", "web3", web3);
-            async.each(commands, ({varName, code}, next) => {
-              if (varName) {
-                engine.events.emit("runcode:register", varName, code);
-              } else {
-                engine.events.request("runcode:eval", code);
-              }
-              next();
-            }, callback);
-          });
-        });
       },
       function deploy(callback) {
         // Skip if we are connected to a websocket, the server will do it
-        if(engine.ipc.connected && engine.ipc.isClient()) {
+        if(isSecondaryProcess(engine)) {
           return callback();
         }
         engine.config.reloadConfig();
@@ -354,7 +329,7 @@ class EmbarkController {
       },
       function waitForWriteFinish(callback) {
         // Skip if we are connected to a websocket, the server will do it
-        if(engine.ipc.connected && engine.ipc.isClient()) {
+        if(isSecondaryProcess(engine)) {
           return callback();
         }
         engine.logger.info("Finished deploying".underline);
@@ -657,7 +632,7 @@ class EmbarkController {
         });
         engine.startService("storage");
         engine.startService("codeGenerator");
-        engine.startService("console", {forceRegister: true});
+        engine.startService("console");
         engine.startService("pluginCommand");
         if (options.coverage) {
           engine.startService("codeCoverage");

packages/embark/src/lib/core/engine.js

@@ -169,15 +169,14 @@ class Engine {
     this.registerModule('plugin_cmd', {embarkConfigFile: this.embarkConfig, embarkConfig: this.config.embarkConfig, packageFile: 'package.json'});
   }
 
-  console(options) {
+  console(_options) {
     this.registerModule('console', {
       events: this.events,
       plugins: this.plugins,
       version: this.version,
       ipc: this.ipc,
       logger: this.logger,
-      config: this.config,
+      config: this.config
-      forceRegister: options.forceRegister
     });
   }
 

packages/embark/src/lib/modules/blockchain_connector/index.js

@@ -192,8 +192,8 @@ class BlockchainConnector {
   }
 
   _emitWeb3Ready() {
-    this.isWeb3Ready = true;
     this.registerWeb3Object(() => {
+      this.isWeb3Ready = true;
       this.events.emit(WEB3_READY);
     });
     this.subscribeToPendingTransactions();
@@ -695,7 +695,7 @@ class BlockchainConnector {
   registerWeb3Object(cb = () => {}) {
     // doesn't feel quite right, should be a cmd or plugin method
     // can just be a command without a callback
-    this.events.emit("runcode:register", "web3", this.web3, false, cb);
+    this.events.emit("runcode:register", "web3", this.web3, cb);
   }
 
   subscribeToPendingTransactions() {

packages/embark/src/lib/modules/codeRunner/index.js

@@ -1,6 +1,5 @@
 const VM = require('./vm');
 const fs = require('../../core/fs');
-const deepEqual = require('deep-equal');
 const EmbarkJS = require('embarkjs');
 const IpfsApi = require("ipfs-api");
 const Web3 = require('web3');
@@ -45,39 +44,12 @@ class CodeRunner {
     this.embark = embark;
     this.commands = [];
 
-    this.registerIpcEvents();
-    this.IpcClientListen();
     this.registerEvents();
     this.registerCommands();
     this.events.emit('runcode:ready');
     this.ready = true;
   }
 
-  registerIpcEvents() {
-    if (!this.ipc.isServer()) {
-      return;
-    }
-
-    this.ipc.on('runcode:getCommands', (_err, callback) => {
-      let result = {web3Config: this.vm.getWeb3Config(), commands: this.commands};
-      callback(null, result);
-    });
-  }
-
-  IpcClientListen() {
-    if (!this.ipc.isClient() || !this.ipc.connected) {
-      return;
-    }
-
-    this.ipc.listenTo('runcode:newCommand', (command) => {
-      if (command.varName) {
-        this.events.emit("runcode:register", command.varName, command.code);
-      } else {
-        this.events.request("runcode:eval", command.code);
-      }
-    });
-  }
-
   registerEvents() {
     this.events.on("runcode:register", this.registerVar.bind(this));
 
@@ -133,37 +105,25 @@ class CodeRunner {
         this.events.request("code-generator:embarkjs:init-provider-code", (providerCode) => {
           this.evalCode(providerCode, (err, _result) => {
             cb(err);
-          }, false, true);
+          }, true);
         });
       }, true);
     });
   }
 
-  registerVar(varName, code, toRecord = true, cb = () => {}) {
+  registerVar(varName, code, cb = () => {}) {
-    const command = {varName, code};
-    if (toRecord && !this.commands.some(cmd => deepEqual(cmd, command))) {
-      if (this.ipc.isServer()) {
-        this.commands.push(command);
-        this.ipc.broadcast("runcode:newCommand", command);
-      }
-    }
     this.vm.registerVar(varName, code, cb);
   }
 
-  evalCode(code, cb, isNotUserInput = false, tolerateError = false) {
+  evalCode(code, cb, tolerateError = false) {
     cb = cb || function () {};
 
     if (!code) return cb(null, '');
 
     this.vm.doEval(code, tolerateError, (err, result) => {
-      if(err) {
+      if (err) {
         return cb(err);
       }
-      const command = {code};
-      if (isNotUserInput && this.ipc.isServer() && !this.commands.some(cmd => cmd.code === command.code)) {
-          this.commands.push(command);
-          this.ipc.broadcast("runcode:newCommand", command);
-      }
       
       cb(null, result);
     });

packages/embark/src/lib/modules/codeRunner/vm.ts

@@ -40,6 +40,8 @@ class VM {
         "@babel/runtime-corejs2/core-js/object/assign",
         "eth-ens-namehash",
         "swarm-api",
+        "rxjs",
+        "rxjs/operators",
       ],
     },
     sandbox: { __dirname: fs.dappPath() },
@@ -102,7 +104,9 @@ class VM {
       if (error.message && error.message.indexOf(WEB3_INVALID_RESPONSE_ERROR) !== -1) {
         error.message += ". Are you connected to an Ethereum node?";
       }
-
+      if (typeof error === "string") {
+        error = new Error(error);
+      }
       return cb(error);
     }
     return cb(null, result);
@@ -151,27 +155,6 @@ class VM {
     this.vm = new NodeVM(this.options);
     cb();
   }
-
-  /**
-   * Gets the registered @type {Web3} object, and returns an @type {object} with it's
-   * defaultAccount and provider URL.
-   * @typedef {getWeb3Config}
-   * @property {string} defaultAccount
-   * @property {string} providerUrl
-   * @returns {getWeb3Config} The configured values of the web3 object registered to this
-   *  VM instance.
-   */
-  public getWeb3Config() {
-    const Web3 = require("web3");
-    const provider = this.options.sandbox.web3.currentProvider;
-    let providerUrl;
-    if (provider instanceof Web3.providers.HttpProvider) {
-      providerUrl = provider.host;
-    } else if (provider instanceof Web3.providers.WebsocketProvider) {
-      providerUrl = provider.connection._url;
-    }
-    return { defaultAccount: this.options.sandbox.web3.eth.defaultAccount, providerUrl };
-  }
 }
 
 module.exports = VM;

packages/embark/src/lib/modules/console/index.ts

@@ -26,9 +26,8 @@ class Console {
   private config: any;
   private history: string[];
   private cmdHistoryFile: string;
-  private suggestions: Suggestions;
+  private suggestions?: Suggestions;
   private providerReady: boolean;
-  private forceRegister: boolean;
 
   constructor(embark: Embark, options: any) {
     this.embark = embark;
@@ -39,14 +38,22 @@ class Console {
     this.fs = embark.fs;
     this.ipc = options.ipc;
     this.config = options.config;
-    this.forceRegister = options.forceRegister;
     this.history = [];
     this.cmdHistoryFile = options.cmdHistoryFile || this.fs.dappPath(".embark", "cmd_history");
     this.providerReady = false;
     this.loadHistory();
 
     if (this.ipc.isServer()) {
-      this.ipc.on("console:executeCmd", this.executeCmd.bind(this));
+      this.ipc.on("console:executeCmd", (cmd: string, cb: any) => {
+        this.executeCmd(cmd, (err: string, result: any) => {
+          let error = null;
+          if (err) {
+            // reformat for IPC reply
+            error = { name: "Console error", message: err, stack: err };
+          }
+          cb(error, result);
+        });
+      });
       this.ipc.on("console:history:save", true, (cmd: string) => {
         this.saveHistory(cmd, true);
       });
@@ -59,6 +66,11 @@ class Console {
       }
       this.events.once("console:provider:done", cb);
     });
+    this.registerConsoleCommands();
+
+    if (this.isEmbarkConsole) {
+      return;
+    }
     this.registerEmbarkJs((err?: Error |  null) => {
       if (err) {
         return this.logger.error(err);
@@ -66,12 +78,15 @@ class Console {
       this.providerReady = true;
       this.events.emit("console:provider:done");
     });
-    this.registerConsoleCommands();
     this.registerApi();
 
     this.suggestions = new Suggestions(embark, options);
   }
 
+  private get isEmbarkConsole() {
+    return this.ipc.connected && this.ipc.isClient();
+  }
+
   private cmdHistorySize() {
     return env.anchoredValue(env.CMD_HISTORY_SIZE);
   }
@@ -165,14 +180,18 @@ class Console {
       return callback(null, output);
     }
 
-    try {
+    // if this is the embark console process, send the command to the process
-      this.events.request("runcode:eval", cmd, callback);
+    // running all the needed services (ie the process running `embark run`)
-    } catch (e) {
+    if (this.isEmbarkConsole) {
-      if (this.ipc.connected && this.ipc.isClient()) {
+      return this.ipc.request("console:executeCmd", cmd, callback);
-        return this.ipc.request("console:executeCmd", cmd, callback);
-      }
-      callback(e);
     }
+
+    this.events.request("runcode:eval", cmd, (err: Error, result: any) => {
+      if (err) {
+        return callback(err.message);
+      }
+      callback(null, result);
+    }, true);
   }
 
   private registerEmbarkJs(cb: Callback<null>) {
@@ -187,19 +206,19 @@ class Console {
           // TODO add docs link to how to install one
           this.logger.warn(__("If you did not install a blockchain connector, stop this process and install one"));
         }, 5000);
-        this.events.once("blockchain:connector:ready", () => {
+        this.events.request("blockchain:connector:ready", () => {
           clearTimeout(waitingForReady);
           next();
         });
       },
       (next: any) => {
+        if (this.isEmbarkConsole) {
+          return next();
+        }
         this.events.request("runcode:blockchain:connected", next);
       },
-      // for every other case (including when asked for force), get the embarkjs
-      // provider code and eval it in the VM (either main running VM or console VM
-      // in the secondary process)
       (next: any) => {
-        if (this.ipc.connected && !this.forceRegister) {
+        if (this.isEmbarkConsole) {
           return next();
         }
         const connectCode = `EmbarkJS.Blockchain.connectConsole((err) => {

packages/embark/src/lib/modules/console/suggestions.ts

@@ -92,7 +92,7 @@ export default class Suggestions {
         }
 
         return cb(this.searchSuggestions(cmd, suggestions));
-      }, false, true);
+      }, true);
     } catch (e) {
     }
 

packages/embark/src/lib/modules/deployment/contract_deployer.js

@@ -211,8 +211,7 @@ class ContractDeployer {
     self.events.emit("deploy:contract:deployed", contract);
 
     self.events.request('code-generator:contract:custom', contract, (contractCode) => {
-      self.events.request('runcode:eval', contractCode, () => {}, true);
+      self.events.request('runcode:eval', contractCode, callback);
-      return callback();
     });
   }
 
@@ -344,7 +343,7 @@ class ContractDeployer {
               self.plugins.runActionsForEvent('deploy:contract:deployed', {contract: contract}, () => {
                 return next(null, receipt);
               });
-            }, true);
+            });
           });
         }, hash => {
           self.logFunction(contract)(__("deploying") + " " + contract.className.bold.cyan + " " + __("with").green + " " + contract.gas + " " + __("gas at the price of").green + " " + contract.gasPrice + " " + __("Wei, estimated cost:").green + " " + estimatedCost + " Wei".green + " (txHash: " + hash.bold.cyan + ")");

packages/embark/src/lib/modules/tests/test.js

@@ -33,20 +33,6 @@ class Test {
       (next) => {
         this.events.request('runcode:ready', next);
       },
-      (next) => {
-        this.initWeb3Provider(next);
-      },
-      (next) => {
-        const waitingForReady = setTimeout(() => {
-          this.logger.warn('Waiting for the blockchain connector to be ready...');
-          // TODO add docs link to how to install one
-          this.logger.warn('If you did not install a blockchain connector, stop this process and install one');
-        }, 5000);
-        this.events.request('blockchain:connector:ready', () => {
-          clearTimeout(waitingForReady);
-          next();
-        });
-      },
       (next) => {
         this.gasLimit = constants.tests.gasLimit;
         this.events.request('deploy:setGasLimit', this.gasLimit);
@@ -185,7 +171,9 @@ class Test {
           return callback(err);
         }
         self.firstRunConfig = false;
-        self.events.request("runcode:embarkjs:reset", callback);
+        self.events.request("blockchain:ready", () => {
+          self.events.request("runcode:embarkjs:reset", callback);
+        });
       });
     });
   }
@@ -346,7 +334,7 @@ class Test {
             Object.setPrototypeOf(self.embarkjs, embarkjs);
           }
           next(err, accounts);
-        }, true);
+        });
       }
     ], function (err, accounts) {
       if (err) {

packages/web3Connector/index.js

@@ -38,19 +38,22 @@ module.exports = async (embark) => {
     if (blockchainConnectorReady) {
       return cb();
     }
-    web3LocationPromise.then((_web3Location) => {
+    embark.events.once("blockchain:connector:ready", () => {
-      blockchainConnectorReady = true;
-      embark.events.emit('blockchain:connector:ready');
       cb();
     });
   });
 
+  web3LocationPromise.then((_web3Location) => {
+    blockchainConnectorReady = true;
+    embark.events.emit('blockchain:connector:ready');
+  });
+
   let web3Location = await web3LocationPromise;
 
   web3Location = web3Location.replace(/\\/g, '/');
 
 
-  embark.events.emit('runcode:register', '__Web3', require(web3Location), false);
+  embark.events.emit('runcode:register', '__Web3', require(web3Location));
 
   let code = `\nconst Web3 = global.__Web3 || require('${web3Location}');`;
   code += `\nglobal.Web3 = Web3;`;
@@ -60,7 +63,7 @@ module.exports = async (embark) => {
 
   code += "\nEmbarkJS.Blockchain.registerProvider('web3', web3Connector);";
 
-  code += "\nEmbarkJS.Blockchain.setProvider('web3', {web3});";
+  code += "\nEmbarkJS.Blockchain.setProvider('web3', {});";
 
   embark.addCodeToEmbarkJS(code);