feat(@embark/embark-rpc-manager): Add support for `eth_signTypedData_v3`

This PR adds support for EIP-712 (https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md), by allowing the signing of typed data transactions using the `eth_signTypeData_v3` or `eth_signTypedData` request.

Add a module called `embark-rpc-manager` to handle all RPC transaction modifications.

The internal events management module extracted to `embark-core`. This allows other modules (ie plugins) to use a private instance of Embark’s event API.

Remove transaction modifications in `embark-accounts-manager` and leave only the functionality in this module that handle account management (ie funding of accounts). Other functionality was moved to `embark-rpc-manager`.

- Transactions should now reflect when a new node account is added via `personal_newAccount` or via the node itself (and once `eth_accounts` is requested).
- In the proxy, errors are handled for all cases and are now JSON-RPC spec-compliant.
- Always register `eth_signTypedData` RPC response and display error message if account cannot be found.

NOTE: Updated yarn.lock due to conflict after rebase.

Avoid race condition with rpc modifications

Refactor proxy to make request handling asynchronous, and to allow circumvention of forwarding requests to the node in cases where it is not needed.

Move loading of plugins to after core and stack components (in cmd_controller).

We should load custom plugins LAST (ie after all core and stack components, and after all Embark plugins). To do this, we disable automatic loading of plugins in the config, and allow them to be loading explicitly in cmd_controller.

Create a system that allows registration of custom plugins. Each plugin is generated in EmbarkJS by instantiating the registered class. For example, we can register a Plasma plugin and it will be generated in to EmbarkJS.Plasma with an instantiation of `embarkjs-plasma`. The DApp itself can then do what it wants with `EmbarkJS.Plasma` (ie call `EmbarkJS.Plasma.init()`.

NOTE: loading of custom plugins needs to be applied to all other commands in `cmd_controller`. This PR only loads custom plugins for the `run` command.

NOTE: This PR is based on branch fix/ws-eth-subscribe (#1850). I will rebase this PR when (and if) #1850 is merged.

packages/core/core/constants.json

@@ -66,7 +66,9 @@
       "eth_getTransactionReceipt": "eth_getTransactionReceipt",
       "eth_call": "eth_call",
       "eth_accounts": "eth_accounts",
-      "personal_listAccounts": "personal_listAccounts"
+      "eth_signTypedData": "eth_signTypedData",
+      "personal_listAccounts": "personal_listAccounts",
+      "personal_newAccount": "personal_newAccount"
     }
   },
   "storage": {

packages/core/core/index.d.ts

@@ -1,5 +0,0 @@
-declare module "embark-core" {
-  export class IPC {
-    constructor(options: {ipcRole: string});
-  }
-}

packages/core/core/src/config.ts

@@ -179,7 +179,6 @@ export class Config {
       env: this.env,
       version: this.version
     });
-    this.plugins.loadPlugins();
 
     this.loadEmbarkConfigFile();
     this.loadBlockchainConfigFile();

packages/core/core/src/engine.ts

@@ -283,6 +283,7 @@ export class Engine {
     this.registerModulePackage('embark-ethereum-blockchain-client');
     this.registerModulePackage('embark-web3');
     this.registerModulePackage('embark-accounts-manager');
+    this.registerModulePackage('embark-rpc-manager');
     this.registerModulePackage('embark-specialconfigs', {plugins: this.plugins});
     this.registerModulePackage('embark-transaction-logger');
     this.registerModulePackage('embark-transaction-tracker');

packages/core/typings/index.d.ts

@@ -1,3 +1,4 @@
+import './src/omg-js-util';
 import './src/prettier-plugin-solidity';
 import './src/remix-debug-debugtest';
 

packages/core/typings/src/omg-js-util/index.d.ts

@@ -0,0 +1 @@
+declare module '@omisego/omg-js-util';

packages/core/utils/src/accountParser.js

@@ -12,6 +12,9 @@ const ERROR_ACCOUNT = 'ERROR_ACCOUNT';
 export default class AccountParser {
   static parseAccountsConfig(accountsConfig, web3, dappPath, logger, nodeAccounts) {
     let accounts = [];
+    if (!(accountsConfig && accountsConfig.length)) {
+      return nodeAccounts;
+    }
     if (accountsConfig && accountsConfig.length) {
       accountsConfig.forEach(accountConfig => {
         let account = AccountParser.getAccount(accountConfig, web3, dappPath, logger, nodeAccounts);

packages/embark/src/cmd/cmd_controller.js

@@ -169,6 +169,9 @@ class EmbarkController {
         engine.registerModuleGroup("cockpit");
         engine.registerModulePackage('embark-deploy-tracker', {plugins: engine.plugins});
 
+        // load custom plugins
+        engine.config.plugins.loadPlugins();
+
         const plugin = engine.plugins.createPlugin('cmdcontrollerplugin', {});
         plugin.registerActionForEvent("embark:engine:started", async (_params, cb) => {
           try {

packages/plugins/accounts-manager/src/index.ts

@@ -1,66 +1,27 @@
 import async from "async";
-import { Embark, Events } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import { Callback, Embark, Events } /* supplied by @types/embark in packages/embark-typings */ from "embark";
 import { __ } from "embark-i18n";
 import { AccountParser, dappPath } from "embark-utils";
 import { Logger } from 'embark-logger';
 import Web3 from "web3";
-const { blockchain: blockchainConstants } = require("embark-core/constants");
 
 import fundAccount from "./fundAccount";
 
-function arrayEqual(arrayA: string[], arrayB: string[]) {
-  if (arrayA.length !== arrayB.length) {
-    return false;
-  } else {
-    return arrayA.every((address, index) => Web3.utils.toChecksumAddress(address) === Web3.utils.toChecksumAddress(arrayB[index]));
-  }
-}
-
 export default class AccountsManager {
   private readonly logger: Logger;
   private readonly events: Events;
-  private accounts: any[] = [];
-  private nodeAccounts: string[] = [];
   private _web3: Web3 | null = null;
-  private ready = false;
+  private _accounts: any[] | null = null;
-  private signTransactionQueue: any;
-  private nonceCache: any = {};
 
   constructor(private readonly embark: Embark, _options: any) {
     this.logger = embark.logger;
     this.events = embark.events;
 
-    this.embark.registerActionForEvent("blockchain:proxy:request", this.checkBlockchainRequest.bind(this));
+    this.parseAndFundAccounts();
-    this.embark.registerActionForEvent("blockchain:proxy:response", this.checkBlockchainResponse.bind(this));
-
     this.events.on("blockchain:started", () => {
       this._web3 = null;
-      this.parseAndFundAccounts(null);
-    });
-    this.embark.registerActionForEvent("accounts:reseted", async (params, cb) => {
-      this.ready = false;
-      await this.parseAndFundAccounts(params.accounts);
-      cb(null, null);
     });
 
-    // Allow to run transaction in parallel by resolving the nonce manually.
-    // For each transaction, resolve the nonce by taking the max of current transaction count and the cache we keep locally.
-    // Update the nonce and sign it
-    this.signTransactionQueue = async.queue(({ payload, account }, callback: (error: any, result: any) => void) => {
-      this.getNonce(payload.from, async (err: any, newNonce: number) => {
-        if (err) {
-          return callback(err, null);
-        }
-        payload.nonce = newNonce;
-        const web3 = await this.web3;
-        web3.eth.accounts.signTransaction(payload, account.privateKey, (signingError: any, result: any) => {
-          if (signingError) {
-            return callback(signingError, null);
-          }
-          callback(null, result.rawTransaction);
-        });
-      });
-    }, 1);
   }
 
   get web3() {
@@ -73,84 +34,27 @@ export default class AccountsManager {
     })();
   }
 
-  private async getNonce(address: string, callback: (error: any, result: any) => void) {
+  get accounts() {
+    return (async () => {
+      if (!this._accounts) {
         const web3 = await this.web3;
-    web3.eth.getTransactionCount(address, undefined, (error: any, transactionCount: number) => {
+        this._accounts = await web3.eth.getAccounts();
-      if (error) {
-        return callback(error, null);
       }
-      if (this.nonceCache[address] === undefined) {
+      return this._accounts || [];
-        this.nonceCache[address] = -1;
+    })();
   }
 
-      if (transactionCount > this.nonceCache[address]) {
+  private async parseAndFundAccounts() {
-        this.nonceCache[address] = transactionCount;
-        return callback(null, this.nonceCache[address]);
-      }
-
-      this.nonceCache[address]++;
-      callback(null, this.nonceCache[address]);
-    });
-  }
-
-  private async checkBlockchainRequest(params: any, callback: (error: any, result: any) => void) {
-    if (!this.ready) {
-      return callback(null, params);
-    }
-    if (params.reqData.method === blockchainConstants.transactionMethods.eth_sendTransaction && this.accounts.length) {
-      // Check if we have that account in our wallet
-      const account = this.accounts.find((acc) => Web3.utils.toChecksumAddress(acc.address) === Web3.utils.toChecksumAddress(params.reqData.params[0].from));
-      if (account && account.privateKey) {
-        return this.signTransactionQueue.push({ payload: params.reqData.params[0], account }, (err: any, newPayload: any) => {
-          if (err) {
-            return callback(err, null);
-          }
-          params.reqData.method = blockchainConstants.transactionMethods.eth_sendRawTransaction;
-          params.reqData.params = [newPayload];
-          callback(err, params);
-        });
-      }
-    }
-    callback(null, params);
-  }
-
-  private async checkBlockchainResponse(params: any, callback: (error: any, result: any) => void) {
-    if (!this.ready) {
-      return callback(null, params);
-    }
-    if ((params.reqData.method === blockchainConstants.transactionMethods.eth_accounts ||
-      params.reqData.method === blockchainConstants.transactionMethods.personal_listAccounts) && this.accounts.length) {
-      if (!arrayEqual(params.respData.result, this.nodeAccounts)) {
-        this.nodeAccounts = params.respData.result;
     const web3 = await this.web3;
-        this.accounts = AccountParser.parseAccountsConfig(this.embark.config.blockchainConfig.accounts, web3, dappPath(), this.logger, this.nodeAccounts);
+    const accounts = await this.accounts;
-      }
-      params.respData.result = this.accounts.map((acc) => {
-        if (acc.address) {
-          return acc.address;
-        }
-        return acc;
-      });
-      return callback(null, params);
-    }
-    callback(null, params);
-  }
 
-  private async parseAndFundAccounts(accounts: any[] | null) {
+    if (!accounts.length || !this.embark.config.blockchainConfig.isDev) {
-    const web3 = await this.web3;
-
-    const nodeAccounts = await web3.eth.getAccounts();
-    this.nodeAccounts = nodeAccounts;
-    this.accounts = AccountParser.parseAccountsConfig(accounts || this.embark.config.blockchainConfig.accounts, web3, dappPath(), this.logger, nodeAccounts);
-
-    if (!this.accounts.length || !this.embark.config.blockchainConfig.isDev) {
-      this.ready = true;
       return;
     }
     try {
       const coinbase = await web3.eth.getCoinbase();
-      const accts = this.accounts
+      const accts = accounts
-        .filter((account) => account.address);
+        .filter((account) => account && account.address);
 
       async.eachLimit(accts, 1, (acct, eachCb) => {
         fundAccount(web3, acct.address, coinbase, acct.hexBalance)
@@ -162,6 +66,5 @@ export default class AccountsManager {
     } catch (err) {
       this.logger.error(__("Error funding accounts"), err.message || err);
     }
-    this.ready = true;
   }
 }

packages/plugins/geth/src/check.js

@@ -7,6 +7,9 @@ const parseAndRespond = (data, cb) => {
   let resp;
   try {
     resp = JSON.parse(data);
+    if (resp.error) {
+      return cb(resp.error);
+    }
   } catch (e) {
     return cb('Version data is not valid JSON');
   }

packages/plugins/rpc-manager/.npmrc

@@ -0,0 +1,4 @@
+engine-strict = true
+package-lock = false
+save-exact = true
+scripts-prepend-node-path = true

packages/plugins/rpc-manager/README.md

@@ -0,0 +1,9 @@
+embark-rpc-manager
+==========================
+
+> Embark RPC Manager
+
+Modifies RPC calls to/from Embark (to/from `embark-proxy`).
+
+Visit [embark.status.im](https://embark.status.im/) to get started with
+[Embark](https://github.com/embark-framework/embark).

packages/plugins/rpc-manager/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "embark-rpc-manager",
+  "version": "5.0.0-alpha.1",
+  "description": "Embark RPC Manager",
+  "main": "./dist/lib/index.js",
+  "repository": {
+    "directory": "packages/plugins/embark-rpc-manager",
+    "type": "git",
+    "url": "https://github.com/embark-framework/embark/"
+  },
+  "author": "Iuri Matias <iuri.matias@gmail.com>",
+  "license": "MIT",
+  "bugs": "https://github.com/embark-framework/embark/issues",
+  "keywords": [
+    "blockchain",
+    "dapps",
+    "ethereum",
+    "ipfs",
+    "serverless",
+    "solc",
+    "solidity"
+  ],
+  "files": [
+    "dist/"
+  ],
+  "embark-collective": {
+    "build:node": true
+  },
+  "scripts": {
+    "_build": "npm run solo -- build",
+    "ci": "npm run qa",
+    "clean": "npm run reset",
+    "lint": "eslint src/",
+    "qa": "npm-run-all lint _build test",
+    "reset": "npx rimraf .nyc_output coverage dist embark-*.tgz package",
+    "solo": "embark-solo",
+    "test": "nyc --reporter=html --reporter=json mocha \"dist/test/**/*.js\" --exit --no-timeouts --require source-map-support/register"
+  },
+  "eslintConfig": {
+    "extends": "../../../.eslintrc.json"
+  },
+  "dependencies": {
+    "@babel/runtime-corejs3": "7.6.3",
+    "@omisego/omg-js-util": "2.0.0-v0.2",
+    "async": "2.6.1",
+    "embark-core": "^5.0.0-alpha.1",
+    "embark-i18n": "^5.0.0-alpha.1",
+    "embark-utils": "^5.0.0-alpha.1",
+    "web3": "1.2.1"
+  },
+  "devDependencies": {
+    "@types/async": "2.0.50",
+    "cross-env": "5.2.0",
+    "eslint": "5.7.0",
+    "mocha": "6.2.0",
+    "npm-run-all": "4.1.5",
+    "rimraf": "3.0.0"
+  },
+  "engines": {
+    "node": ">=10.17.0 <12.0.0",
+    "npm": ">=6.11.3",
+    "yarn": ">=1.19.1"
+  },
+  "nyc": {
+    "exclude": [
+      "**/node_modules/**",
+      "coverage/**",
+      "dist/test/**"
+    ]
+  }
+}

packages/plugins/rpc-manager/src/lib/eth_accounts.ts

@@ -0,0 +1,54 @@
+import { Callback, Embark, Events, Logger } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import Web3 from "web3";
+const { blockchain: blockchainConstants } = require("embark-core/constants");
+import { __ } from "embark-i18n";
+import RpcModifier from "./rpcModifier";
+
+const METHODS_TO_MODIFY = [
+  blockchainConstants.transactionMethods.eth_accounts,
+  blockchainConstants.transactionMethods.personal_listAccounts,
+];
+
+function arrayEqual(arrayA: string[], arrayB: string[]) {
+  if (!(arrayA && arrayB) || arrayA.length !== arrayB.length) {
+    return false;
+  } else {
+    return arrayA.every((address, index) => Web3.utils.toChecksumAddress(address) === Web3.utils.toChecksumAddress(arrayB[index]));
+  }
+}
+
+export default class EthAccounts extends RpcModifier {
+  constructor(embark: Embark, rpcModifierEvents: Events) {
+    super(embark, rpcModifierEvents);
+
+    this.embark.registerActionForEvent("blockchain:proxy:response", this.checkResponseFor_eth_accounts.bind(this));
+  }
+
+  private async checkResponseFor_eth_accounts(params: any, callback: Callback<any>) {
+
+    if (!(METHODS_TO_MODIFY.includes(params.reqData.method))) {
+      return callback(null, params);
+    }
+
+    this.logger.trace(__(`Modifying blockchain '${params.reqData.method}' response:`));
+    this.logger.trace(__(`Original request/response data: ${JSON.stringify(params)}`));
+
+    try {
+      if (!arrayEqual(params.respData.result, this._nodeAccounts || [])) {
+        // reset backing variables so accounts is recalculated
+        await this.rpcModifierEvents.request2("nodeAccounts:updated", params.respData.result);
+      }
+      const accounts = await this.accounts;
+      if (!(accounts && accounts.length)) {
+        return callback(null, params);
+      }
+
+      params.respData.result = accounts.map((acc) => acc.address || acc);
+      this.logger.trace(__(`Modified request/response data: ${JSON.stringify(params)}`));
+    } catch (err) {
+      return callback(err);
+    }
+
+    return callback(null, params);
+  }
+}

packages/plugins/rpc-manager/src/lib/eth_sendTransaction.ts

@@ -0,0 +1,86 @@
+import async from "async";
+import { Callback, Embark, Events, Logger } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import { __ } from "embark-i18n";
+import Web3 from "web3";
+const { blockchain: blockchainConstants } = require("embark-core/constants");
+import RpcModifier from "./rpcModifier";
+
+export default class EthSendTransaction extends RpcModifier {
+  private signTransactionQueue: any;
+  private nonceCache: any = {};
+  constructor(embark: Embark, rpcModifierEvents: Events) {
+    super(embark, rpcModifierEvents);
+
+    embark.registerActionForEvent("blockchain:proxy:request", this.checkRequestFor_eth_sendTransaction.bind(this));
+
+    // Allow to run transaction in parallel by resolving the nonce manually.
+    // For each transaction, resolve the nonce by taking the max of current transaction count and the cache we keep locally.
+    // Update the nonce and sign it
+    this.signTransactionQueue = async.queue(({ payload, account }, callback: Callback<any>) => {
+      this.getNonce(payload.from, async (err: any, newNonce: number) => {
+        if (err) {
+          return callback(err, null);
+        }
+        payload.nonce = newNonce;
+        const web3 = await this.web3;
+        web3.eth.accounts.signTransaction(payload, account.privateKey, (signingError: any, result: any) => {
+          if (signingError) {
+            return callback(signingError, null);
+          }
+          callback(null, result.rawTransaction);
+        });
+      });
+    }, 1);
+  }
+
+  private async getNonce(address: string, callback: Callback<any>) {
+    const web3 = await this.web3;
+    web3.eth.getTransactionCount(address, undefined, (error: any, transactionCount: number) => {
+      if (error) {
+        return callback(error, null);
+      }
+      if (this.nonceCache[address] === undefined) {
+        this.nonceCache[address] = -1;
+      }
+
+      if (transactionCount > this.nonceCache[address]) {
+        this.nonceCache[address] = transactionCount;
+        return callback(null, this.nonceCache[address]);
+      }
+
+      this.nonceCache[address]++;
+      callback(null, this.nonceCache[address]);
+    });
+  }
+  private async checkRequestFor_eth_sendTransaction(params: any, callback: Callback<any>) {
+    if (!(params.reqData.method === blockchainConstants.transactionMethods.eth_sendTransaction)) {
+      return callback(null, params);
+    }
+    const accounts = await this.accounts;
+    if (!(accounts && accounts.length)) {
+      return callback(null, params);
+    }
+
+    this.logger.trace(__(`Modifying blockchain '${params.reqData.method}' request:`));
+    this.logger.trace(__(`Original request data: ${JSON.stringify(params)}`));
+
+    try {
+      // Check if we have that account in our wallet
+      const account = accounts.find((acc) => Web3.utils.toChecksumAddress(acc.address) === Web3.utils.toChecksumAddress(params.reqData.params[0].from));
+      if (account && account.privateKey) {
+        return this.signTransactionQueue.push({ payload: params.reqData.params[0], account }, (err: any, newPayload: any) => {
+          if (err) {
+            return callback(err, null);
+          }
+          params.reqData.method = blockchainConstants.transactionMethods.eth_sendRawTransaction;
+          params.reqData.params = [newPayload];
+          callback(err, params);
+        });
+      }
+    } catch (err) {
+      return callback(err);
+    }
+    this.logger.trace(__(`Modified request/response data: ${JSON.stringify(params)}`));
+    callback(null, params);
+  }
+}

packages/plugins/rpc-manager/src/lib/eth_signTypedData.ts

@@ -0,0 +1,61 @@
+import { sign, transaction } from "@omisego/omg-js-util";
+import { Callback, Embark, Events, Logger } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import { __ } from "embark-i18n";
+import Web3 from "web3";
+import RpcModifier from "./rpcModifier";
+
+export default class EthSignTypedData extends RpcModifier {
+  constructor(embark: Embark, rpcModifierEvents: Events) {
+    super(embark, rpcModifierEvents);
+
+    this.embark.registerActionForEvent("blockchain:proxy:request", this.checkRequestFor_eth_signTypedData.bind(this));
+    this.embark.registerActionForEvent("blockchain:proxy:response", this.checkResponseFor_eth_signTypedData.bind(this));
+  }
+
+  private async checkRequestFor_eth_signTypedData(params: any, callback: Callback<any>) {
+    // check for:
+    // - eth_signTypedData
+    // - eth_signTypedData_v3
+    // - eth_signTypedData_v4
+    // - personal_signTypedData (parity)
+    if (params.reqData.method.includes("signTypedData")) {
+      // indicate that we do not want this call to go to the node
+      params.sendToNode = false;
+      return callback(null, params);
+    }
+    callback(null, params);
+  }
+  private async checkResponseFor_eth_signTypedData(params: any, callback: Callback<any>) {
+
+    // check for:
+    // - eth_signTypedData
+    // - eth_signTypedData_v3
+    // - eth_signTypedData_v4
+    // - personal_signTypedData (parity)
+    if (!params.reqData.method.includes("signTypedData")) {
+      return callback(null, params);
+    }
+
+    this.logger.trace(__(`Modifying blockchain '${params.reqData.method}' response:`));
+    this.logger.trace(__(`Original request/response data: ${JSON.stringify(params)}`));
+
+    try {
+      const accounts = await this.accounts;
+      const [fromAddr, typedData] = params.reqData.params;
+      const account = accounts.find((acc) => Web3.utils.toChecksumAddress(acc.address) === Web3.utils.toChecksumAddress(fromAddr));
+      if (!(account && account.privateKey)) {
+        return callback(
+          new Error(__("Could not sign transaction because Embark does not have a private key associated with '%s'. " +
+            "Please ensure you have configured your account(s) to use a mnemonic, privateKey, or privateKeyFile.", fromAddr)));
+      }
+      const toSign = transaction.getToSignHash(typeof typedData === "string" ? JSON.parse(typedData) : typedData);
+      const signature = sign(toSign, [account.privateKey]);
+
+      params.respData.result = signature[0];
+      this.logger.trace(__(`Modified request/response data: ${JSON.stringify(params)}`));
+    } catch (err) {
+      return callback(err);
+    }
+    callback(null, params);
+  }
+}

packages/plugins/rpc-manager/src/lib/index.ts

@@ -0,0 +1,45 @@
+import { Callback, Embark, Events } from "embark-core";
+import { Logger } from 'embark-logger';
+import Web3 from "web3";
+import EthAccounts from "./eth_accounts";
+import EthSendTransaction from "./eth_sendTransaction";
+import EthSignTypedData from "./eth_signTypedData";
+import PersonalNewAccount from "./personal_newAccount";
+import RpcModifier from "./rpcModifier";
+
+export default class RpcManager {
+
+  private modifiers: RpcModifier[] = [];
+  private _web3: Web3 | null = null;
+  private rpcModifierEvents: Events;
+  private logger: Logger;
+  private events: Events;
+  public _accounts: any[] | null = null;
+  public _nodeAccounts: any[] | null = null;
+  constructor(private readonly embark: Embark) {
+    this.events = embark.events;
+    this.logger = embark.logger;
+    this.rpcModifierEvents = new Events();
+    this.init();
+  }
+
+  private async init() {
+
+    this.rpcModifierEvents.setCommandHandler("nodeAccounts:updated", this.updateAccounts.bind(this));
+    this.rpcModifierEvents.setCommandHandler("nodeAccounts:added", async (addedNodeAccount: any, cb: Callback<null>) => {
+      if (!this._nodeAccounts) {
+        this._nodeAccounts = [addedNodeAccount];
+      } else {
+        this._nodeAccounts.push(addedNodeAccount);
+      }
+      return this.updateAccounts(this._nodeAccounts, cb);
+    });
+    this.modifiers = [PersonalNewAccount, EthAccounts, EthSendTransaction, EthSignTypedData].map((rpcModifier) => new rpcModifier(this.embark, this.rpcModifierEvents));
+  }
+  private async updateAccounts(updatedNodeAccounts: any[], cb: Callback<null>) {
+    for (const modifier of this.modifiers) {
+      await (modifier.nodeAccounts = Promise.resolve(updatedNodeAccounts));
+    }
+    cb();
+  }
+}

packages/plugins/rpc-manager/src/lib/personal_newAccount.ts

@@ -0,0 +1,23 @@
+import { Callback, Embark, Events } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import Web3 from "web3";
+const { blockchain: blockchainConstants } = require("embark-core/constants");
+import { __ } from "embark-i18n";
+import RpcModifier from "./rpcModifier";
+export default class PersonalNewAccount extends RpcModifier {
+  constructor(embark: Embark, rpcModifierEvents: Events) {
+    super(embark, rpcModifierEvents);
+
+    embark.registerActionForEvent("blockchain:proxy:response", this.checkResponseFor_personal_newAccount.bind(this));
+  }
+
+  private async checkResponseFor_personal_newAccount(params: any, callback: Callback<any>) {
+    if (params.reqData.method !== blockchainConstants.transactionMethods.personal_newAccount) {
+      return callback(null, params);
+    }
+
+    // emit event so tx modifiers can refresh accounts
+    await this.rpcModifierEvents.request2("nodeAccounts:added", params.respData.result);
+
+    callback(null, params);
+  }
+}

packages/plugins/rpc-manager/src/lib/rpcModifier.ts

@@ -0,0 +1,60 @@
+import { Embark, Events, Logger } /* supplied by @types/embark in packages/embark-typings */ from "embark";
+import { AccountParser, dappPath } from "embark-utils";
+import Web3 from "web3";
+
+export default class RpcModifier {
+  public events: Events;
+  public logger: Logger;
+  private _web3: Web3 | null = null;
+  private _accounts: any[] | null = null;
+  protected _nodeAccounts: any[] | null = null;
+  constructor(readonly embark: Embark, readonly rpcModifierEvents: Events) {
+    this.events = embark.events;
+    this.logger = embark.logger;
+  }
+
+  protected get web3() {
+    return (async () => {
+      if (!this._web3) {
+        const provider = await this.events.request2("blockchain:client:provider", "ethereum");
+        this._web3 = new Web3(provider);
+      }
+      return this._web3;
+    })();
+  }
+
+  public get nodeAccounts() {
+    return (async () => {
+      if (!this._nodeAccounts) {
+        const web3 = await this.web3;
+        this._nodeAccounts = await web3.eth.getAccounts();
+      }
+      return this._nodeAccounts || [];
+    })();
+  }
+
+  public set nodeAccounts(nodeAccounts: Promise<any[]>) {
+    (async () => {
+      this._nodeAccounts = await nodeAccounts;
+      // reset accounts backing variable as it neesd to be recalculated
+      this._accounts = null;
+    })();
+  }
+
+  protected get accounts() {
+    return (async () => {
+      if (!this._accounts) {
+        const web3 = await this.web3;
+        const nodeAccounts = await this.nodeAccounts;
+        this._accounts = AccountParser.parseAccountsConfig(this.embark.config.blockchainConfig.accounts, web3, dappPath(), this.logger, nodeAccounts);
+      }
+      return this._accounts || [];
+    })();
+  }
+
+  protected set accounts(accounts) {
+    (async () => {
+      this._accounts = await accounts;
+    })();
+  }
+}

packages/plugins/rpc-manager/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../../tsconfig.json",
+  "include": ["src/**/*"]
+}

packages/plugins/rpc-manager/tslint.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../../tslint.json"
+}

packages/stack/embarkjs/src/embarkjs-artifact.js.ejs

@@ -52,4 +52,13 @@ if (typeof WebSocket !== 'undefined') {
 }
 <% } %>
 
+<% for (let stackName in (customPlugins || [])) { %>
+  <% for (let pluginName in (customPlugins[stackName] || [])) { %>
+    let __embark<%- pluginName %> = require('<%- customPlugins[stackName][pluginName] %>');
+    __embark<%- pluginName %> = __embark<%- pluginName %>.default || __embark<%- pluginName %>;
+    const customPluginConfig = require('./config/<%- stackName %>.json');
+    EmbarkJS.<%- stackName %> = new __embark<%- pluginName %>({pluginConfig: customPluginConfig});
+  <% }; %>
+<% }; %>
+
 export default EmbarkJS;

packages/stack/embarkjs/src/index.js

@@ -21,16 +21,26 @@ class EmbarkJS {
     });
 
     this.embarkJSPlugins = {};
+    this.customEmbarkJSPlugins = {};
     this.events.setCommandHandler("embarkjs:plugin:register", (stackName, pluginName, packageName) => {
       this.embarkJSPlugins[stackName] = this.embarkJSPlugins[stackName] || {};
       this.embarkJSPlugins[stackName][pluginName] = packageName;
     });
+    this.events.setCommandHandler("embarkjs:plugin:register:custom", (stackName, pluginName, packageName) => {
+      this.customEmbarkJSPlugins[stackName] = this.customEmbarkJSPlugins[stackName] || {};
+      this.customEmbarkJSPlugins[stackName][pluginName] = packageName;
+    });
 
     this.events.setCommandHandler("embarkjs:console:register", (stackName, pluginName, packageName, cb) => {
       this.events.request("runcode:whitelist", packageName, () => {});
       this.registerEmbarkJSPlugin(stackName, pluginName, packageName, cb || (() => {}));
     });
 
+    this.events.setCommandHandler("embarkjs:console:regsiter:custom", (stackName, pluginName, packageName, options, cb) => {
+      this.events.request("runcode:whitelist", packageName, () => {});
+      this.registerCustomEmbarkJSPluginInVm(stackName, pluginName, packageName, options, cb || (() => {}));
+    });
+
     this.events.setCommandHandler("embarkjs:console:setProvider", this.setProvider.bind(this));
     this.events.setCommandHandler("embarkjs:contract:generate", this.addContractArtifact.bind(this));
     this.events.setCommandHandler("embarkjs:contract:runInVm", this.runInVm.bind(this));
@@ -65,8 +75,27 @@ class EmbarkJS {
     cb();
   }
 
+  async registerCustomEmbarkJSPluginInVm(stackName, pluginName, packageName, options, cb) {
+    await this.registerEmbarkJS();
+
+    const customPluginCode = `
+      let __embark${pluginName} = require('${packageName}');
+      __embark${pluginName} = __embark${pluginName}.default || __embark${pluginName};
+      const customPluginOptions = ${JSON.stringify(options)};
+      EmbarkJS.${stackName} = new __embark${pluginName}({pluginConfig: customPluginOptions});
+      EmbarkJS.${stackName}.init();
+    `;
+
+    await this.events.request2('runcode:eval', customPluginCode);
+    cb();
+  }
+
   addEmbarkJSArtifact(_params, cb) {
-    const embarkjsCode = Templates.embarkjs_artifact({ plugins: this.embarkJSPlugins, hasWebserver: this.embark.config.webServerConfig.enabled });
+    const embarkjsCode = Templates.embarkjs_artifact({
+      plugins: this.embarkJSPlugins,
+      hasWebserver: this.embark.config.webServerConfig.enabled,
+      customPlugins: this.customEmbarkJSPlugins
+    });
 
     // TODO: generate a .node file
     this.events.request("pipeline:register", {
@@ -142,10 +171,10 @@ class EmbarkJS {
       const result = await this.events.request2('runcode:eval', contract.className);
       result.currentProvider = provider;
       await this.events.request2("runcode:register", contract.className, result);
-      cb();
     } catch (err) {
-      cb(err);
+      return cb(err);
     }
+    cb();
   }
 
 }

packages/stack/proxy/src/index.ts

@@ -9,7 +9,8 @@ const constants = require("embark-core/constants");
 export default class ProxyManager {
   private readonly logger: Logger;
   private readonly events: Events;
-  private proxy: any;
+  private wsProxy: any;
+  private httpProxy: any;
   private plugins: any;
   private readonly host: string;
   private rpcPort = 0;
@@ -79,7 +80,7 @@ export default class ProxyManager {
     if (!this.embark.config.blockchainConfig.proxy) {
       return;
     }
-    if (this.proxy) {
+    if (this.httpProxy || this.wsProxy) {
       throw new Error("Proxy is already started");
     }
     const port = await findNextPort(this.embark.config.blockchainConfig.rpcPort + constants.blockchain.servicePortOnProxy);
@@ -88,23 +89,47 @@ export default class ProxyManager {
     this.wsPort = port + 1;
     this.isWs = clientName === constants.blockchain.vm || (/wss?/).test(this.embark.config.blockchainConfig.endpoint);
 
-    this.proxy = await new Proxy({
+    // HTTP
-      endpoint: clientName === constants.blockchain.vm ? constants.blockchain.vm : this.embark.config.blockchainConfig.endpoint,
+    if (clientName !== constants.blockchain.vm) {
+      this.httpProxy = await new Proxy({
+        endpoint: this.embark.config.blockchainConfig.endpoint,
         events: this.events,
-      isWs: this.isWs,
+        isWs: false,
         logger: this.logger,
         plugins: this.plugins,
         vms: this.vms,
-    });
+      })
-
+      .serve(
-    await this.proxy.serve(
         this.host,
-      this.isWs ? this.wsPort : this.rpcPort,
+        this.rpcPort,
       );
+      this.logger.info(`HTTP Proxy for node endpoint ${this.embark.config.blockchainConfig.endpoint} listening on ${buildUrl("http", this.host, this.rpcPort, "rpc")}`);
+    }
+    if (this.isWs) {
+      const endpoint = clientName === constants.blockchain.vm ? constants.blockchain.vm : this.embark.config.blockchainConfig.endpoint;
+      this.wsProxy = await new Proxy({
+        endpoint,
+        events: this.events,
+        isWs: true,
+        logger: this.logger,
+        plugins: this.plugins,
+        vms: this.vms,
+      })
+      .serve(
+        this.host,
+        this.wsPort,
+      );
+      this.logger.info(`WS Proxy for node endpoint ${endpoint} listening on ${buildUrl("ws", this.host, this.wsPort, "ws")}`);
+    }
   }
-
   private stopProxy() {
-    this.proxy.stop();
+    if (this.wsProxy) {
-    this.proxy = null;
+      this.wsProxy.stop();
+      this.wsProxy = null;
+    }
+    if (this.httpProxy) {
+      this.httpProxy.stop();
+      this.httpProxy = null;
+    }
   }
 }

packages/stack/proxy/src/proxy.js

@@ -173,6 +173,7 @@ export class Proxy {
         // stripped out by modifying the response (via actions for blockchain:proxy:response)
         respData.error = fwdReqErr.message || fwdReqErr;
       }
+    }
 
     try {
       const modifiedResp = await this.emitActionsForResponse(modifiedRequest.reqData, respData);
@@ -193,7 +194,7 @@ export class Proxy {
       const rpcErrorObj = { "jsonrpc": "2.0", "error": { "code": -32603, "message": error }, "id": modifiedRequest.reqData.id };
       return this.respondError(transport, rpcErrorObj);
     }
-    }
+
 
 
   }

yarn.lock

@@ -2598,6 +2598,17 @@
     universal-user-agent "^3.0.0"
     url-template "^2.0.8"
 
+"@omisego/omg-js-util@2.0.0-v0.2":
+  version "2.0.0-v0.2"
+  resolved "https://registry.yarnpkg.com/@omisego/omg-js-util/-/omg-js-util-2.0.0-v0.2.tgz#30b5882cc8a45446e7206576240caade614c8590"
+  integrity sha512-J5J5Q2XC7ZkpfQJiXJxii/66Tn3x4rhnhky3BmL++XEBoK6I34wgWMs/7dzsejWNHmmhGv/M/OZb+QBI+HlvlA==
+  dependencies:
+    eth-sig-util "^2.1.1"
+    ethereumjs-util "^6.0.0"
+    js-sha3 "^0.8.0"
+    number-to-bn "^1.7.0"
+    rlp "^2.2.2"
+
 "@reach/router@^1.2.1":
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/@reach/router/-/router-1.2.1.tgz#34ae3541a5ac44fa7796e5506a5d7274a162be4e"
@@ -5062,7 +5073,7 @@ bn.js@4.11.6:
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.6.tgz#53344adb14617a13f6e8dd2ce28905d1c0ba3215"
   integrity sha1-UzRK2xRhehP26N0s4okF0cC6MhU=
 
-bn.js@4.11.8, bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.0, bn.js@^4.11.1, bn.js@^4.11.6, bn.js@^4.11.8, bn.js@^4.4.0, bn.js@^4.8.0:
+bn.js@4.11.8, bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.10.0, bn.js@^4.11.0, bn.js@^4.11.1, bn.js@^4.11.6, bn.js@^4.11.8, bn.js@^4.4.0, bn.js@^4.8.0:
   version "4.11.8"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"
   integrity sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA==
@@ -8466,6 +8477,18 @@ eth-lib@0.2.7:
     elliptic "^6.4.0"
     xhr-request-promise "^0.1.2"
 
+eth-sig-util@^2.1.1:
+  version "2.4.4"
+  resolved "https://registry.yarnpkg.com/eth-sig-util/-/eth-sig-util-2.4.4.tgz#8804ead83de8648bcf81eadbfac1e3ccdd360aea"
+  integrity sha512-iWGqEJwsUMgtk8AqQQqIDTjMz+pW8s2Sq8gN640dh9U9HoEFQJO3m6ro96DgV6hMB2LYu8F5812LQyynOgCbEw==
+  dependencies:
+    buffer "^5.2.1"
+    elliptic "^6.4.0"
+    ethereumjs-abi "0.6.5"
+    ethereumjs-util "^5.1.1"
+    tweetnacl "^1.0.0"
+    tweetnacl-util "^0.15.0"
+
 ethashjs@~0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/ethashjs/-/ethashjs-0.0.7.tgz#30bfe4196726690a0c59d3b8272e70d4d0c34bae"
@@ -8493,6 +8516,14 @@ ethereum-common@^0.0.18:
   resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f"
   integrity sha1-L9w1dvIykDNYl26znaeDIT/5Uj8=
 
+ethereumjs-abi@0.6.5:
+  version "0.6.5"
+  resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241"
+  integrity sha1-WmN+8Wq0NHP6cqKa2QhxQFs/UkE=
+  dependencies:
+    bn.js "^4.10.0"
+    ethereumjs-util "^4.3.0"
+
 ethereumjs-account@^2.0.3:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/ethereumjs-account/-/ethereumjs-account-2.0.5.tgz#eeafc62de544cb07b0ee44b10f572c9c49e00a84"
@@ -8591,7 +8622,7 @@ ethereumjs-util@6.0.0, ethereumjs-util@~6.0.0:
     safe-buffer "^5.1.1"
     secp256k1 "^3.0.1"
 
-ethereumjs-util@^4.0.1, ethereumjs-util@^4.5.0:
+ethereumjs-util@^4.0.1, ethereumjs-util@^4.3.0, ethereumjs-util@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-4.5.0.tgz#3e9428b317eebda3d7260d854fddda954b1f1bc6"
   integrity sha1-PpQosxfuvaPXJg2FT93alUsfG8Y=
@@ -8602,7 +8633,7 @@ ethereumjs-util@^4.0.1, ethereumjs-util@^4.5.0:
     rlp "^2.0.0"
     secp256k1 "^3.0.1"
 
-ethereumjs-util@^5.0.0, ethereumjs-util@^5.1.2, ethereumjs-util@^5.2.0:
+ethereumjs-util@^5.0.0, ethereumjs-util@^5.1.1, ethereumjs-util@^5.1.2, ethereumjs-util@^5.2.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-5.2.0.tgz#3e0c0d1741471acf1036052d048623dee54ad642"
   integrity sha512-CJAKdI0wgMbQFLlLRtZKGcy/L6pzVRgelIZqRqNbuVFM3K9VEnyfbcvz0ncWMRNCe4kaHWjwRYQcYMucmwsnWA==
@@ -14263,7 +14294,7 @@ number-is-nan@^1.0.0:
   resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
   integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
 
-number-to-bn@1.7.0:
+number-to-bn@1.7.0, number-to-bn@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/number-to-bn/-/number-to-bn-1.7.0.tgz#bb3623592f7e5f9e0030b1977bd41a0c53fe1ea0"
   integrity sha1-uzYjWS9+X54AMLGXe9QaDFP+HqA=
@@ -17739,7 +17770,7 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-rlp@^2.0.0:
+rlp@^2.0.0, rlp@^2.2.2:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/rlp/-/rlp-2.2.3.tgz#7f94aef86cec412df87d5ea1d8cb116a47d45f0e"
   integrity sha512-l6YVrI7+d2vpW6D6rS05x2Xrmq8oW7v3pieZOJKBEdjuTF4Kz/iwk55Zyh1Zaz+KOB2kC8+2jZlp2u9L4tTzCQ==
@@ -19847,6 +19878,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+tweetnacl-util@^0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/tweetnacl-util/-/tweetnacl-util-0.15.0.tgz#4576c1cee5e2d63d207fee52f1ba02819480bc75"
+  integrity sha1-RXbBzuXi1j0gf+5S8boCgZSAvHU=
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"