Address Derivation (CI) (#529)

* improve derivation-checking performance by batching docker calls; move into spec dir

* remove npm command to run derivation-checking; create 'int-test' (integration) command and hook up into jest

* add integration testing to CI; configure docker / docker image (dternyak/eth-priv-to-addr) in CI

* docker build -> docker pull

* use travis build matrix to group tests and improve build times

* remove int-test call

* attempt travis 'job' with all tests running in parallel

* remove typo

* attempt travis 'job' with all tests running in parallel (round 2)

* organize integration tests

* refactor/cleanup

* refactor/address comments

.travis.yml

@@ -2,22 +2,34 @@ dist: trusty
 sudo: required
 language: node_js
 
+services:
+  - docker
+
 before_install:
   - export CHROME_BIN=chromium-browser
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
+  - docker pull dternyak/eth-priv-to-addr:latest
 
 install:
   - npm install --silent
 
+jobs:
+  include:
+    - stage: test
+      script: npm run test
+    - stage: test
+      script: npm run test:int
+    - stage: test
+      script: npm run tslint
+    - stage: test
+      script: npm run tscheck
+    - stage: test
+      script: npm run freezer
+    - stage: test
+      script: npm run freezer:validate
+
 notifications:
   email:
     on_success: never
-    on_failure: never
+    on_failure: never
-
-script:
-- npm run test
-- npm run tslint
-- npm run tscheck
-- npm run freezer
-- npm run freezer:validate

common/derivation-checker.ts

@@ -1,57 +0,0 @@
-import assert from 'assert';
-import { generate, IFullWallet } from 'ethereumjs-wallet';
-const { exec } = require('child_process');
-const ProgressBar = require('progress');
-
-// FIXME pick a less magic number
-const derivationRounds = 100;
-const dockerImage = 'dternyak/eth-priv-to-addr';
-const dockerTag = 'latest';
-const bar = new ProgressBar(':percent :bar', { total: derivationRounds });
-
-function promiseFromChildProcess(command): Promise<any> {
-  return new Promise((resolve, reject) => {
-    return exec(command, (err, stdout) => {
-      err ? reject(err) : resolve(stdout);
-    });
-  });
-}
-
-async function privToAddrViaDocker(privKeyWallet: IFullWallet) {
-  const command = `docker run -e key=${privKeyWallet.getPrivateKeyString()} ${
-    dockerImage
-  }:${dockerTag}`;
-  const dockerOutput = await promiseFromChildProcess(command);
-  const newlineStrippedDockerOutput = dockerOutput.replace(
-    /(\r\n|\n|\r)/gm,
-    ''
-  );
-  return newlineStrippedDockerOutput;
-}
-
-async function testDerivation() {
-  const privKeyWallet = generate();
-  const privKeyWalletAddress = await privKeyWallet.getAddressString();
-  const dockerAddr = await privToAddrViaDocker(privKeyWallet);
-  // strip the checksum
-  const lowerCasedPrivKeyWalletAddress = privKeyWalletAddress.toLowerCase();
-  // ensure that pyethereum privToAddr derivation matches our (js based) derivation
-  assert.strictEqual(dockerAddr, lowerCasedPrivKeyWalletAddress);
-}
-
-async function testDerivationNTimes(n = derivationRounds) {
-  let totalRounds = 0;
-  while (totalRounds < n) {
-    await testDerivation();
-    bar.tick();
-    totalRounds += 1;
-  }
-}
-
-console.log('Starting testing...');
-console.time('testDerivationNTimes');
-testDerivationNTimes().then(() => {
-  console.timeEnd('testDerivationNTimes');
-  console.log(`Succeeded testing derivation ${derivationRounds} times :)`);
-  process.exit(0);
-});

jest_config/jest.int.config.json

@@ -0,0 +1,22 @@
+{
+  "rootDir": "../",
+  "transform": {
+    "^.+\\.tsx?$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
+  },
+  "testRegex": "(/__tests__/.*|(\\.|/)(int))\\.(jsx?|tsx?)$",
+  "moduleDirectories": ["node_modules", "common"],
+  "moduleFileExtensions": ["ts", "tsx", "js", "jsx", "json"],
+  "moduleNameMapper": {
+    "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$":
+    "<rootDir>/jest_config/__mocks__/fileMock.ts",
+    "\\.(css|scss|less)$": "<rootDir>/jest_config/__mocks__/styleMock.ts"
+  },
+  "testPathIgnorePatterns": ["<rootDir>/common/config"],
+  "setupFiles": [
+    "<rootDir>/jest_config/setupJest.js",
+    "<rootDir>/jest_config/__mocks__/localStorage.ts"
+  ],
+  "automock": false,
+  "snapshotSerializers": ["enzyme-to-json/serializer"],
+  "browser": true
+}

package.json

@@ -120,13 +120,14 @@
     "build:demo": "BUILD_GH_PAGES=true webpack --config webpack_config/webpack.prod.js",
     "prebuild:demo": "check-node-version --package",
     "test": "jest --config=jest_config/jest.config.json --coverage",
+    "test:unit": "jest --config=jest_config/jest.config.json --coverage",
+    "test:int": "jest --config=jest_config/jest.int.config.json --coverage",
     "updateSnapshot": "jest --config=jest_config/jest.config.json --updateSnapshot",
     "pretest": "check-node-version --package",
     "dev": "node webpack_config/server.js",
     "predev": "check-node-version --package",
     "dev:https": "HTTPS=true node webpack_config/server.js",
     "predev:https": "check-node-version --package",
-    "derivation-checker": "webpack --config=./webpack_config/webpack.derivation-checker.js && node ./dist/derivation-checker.js",
     "tslint": "tslint --project . --exclude common/vendor/**/*",
     "tscheck": "tsc --noEmit",
     "postinstall": "webpack --config=./webpack_config/webpack.dll.js",

spec/integration/data.int.ts

@@ -55,9 +55,7 @@ function testRpcRequests(node: RPCNode, service: string) {
       it(
         `RPC: ${testType} ${service}`,
         () => {
-          return RPCTests[testType](node).then(d =>
+          return RPCTests[testType](node).then(d => expect(d.valid).toBeTruthy());
-            expect(d.valid).toBeTruthy()
-          );
         },
         10000
       );
@@ -69,24 +67,15 @@ const mapNodeEndpoints = (nodes: { [key: string]: NodeConfig }) => {
   const { RpcNodes, EtherscanNodes, InfuraNodes } = RpcNodeTestConfig;
 
   RpcNodes.forEach(n => {
-    testRpcRequests(
+    testRpcRequests(nodes[n].lib as RPCNode, `${nodes[n].service} ${nodes[n].network}`);
-      nodes[n].lib as RPCNode,
-      `${nodes[n].service} ${nodes[n].network}`
-    );
   });
 
   EtherscanNodes.forEach(n => {
-    testRpcRequests(
+    testRpcRequests(nodes[n].lib as EtherscanNode, `${nodes[n].service} ${nodes[n].network}`);
-      nodes[n].lib as EtherscanNode,
-      `${nodes[n].service} ${nodes[n].network}`
-    );
   });
 
   InfuraNodes.forEach(n => {
-    testRpcRequests(
+    testRpcRequests(nodes[n].lib as InfuraNode, `${nodes[n].service} ${nodes[n].network}`);
-      nodes[n].lib as InfuraNode,
-      `${nodes[n].service} ${nodes[n].network}`
-    );
   });
 };
 

spec/integration/derivationChecker.int.ts

@@ -0,0 +1,75 @@
+import { generate, IFullWallet } from 'ethereumjs-wallet';
+import { stripHexPrefix } from '../../common/libs/values';
+const { exec } = require('child_process');
+
+// FIXME pick a less magic number
+const derivationRounds = 500;
+const dockerImage = 'dternyak/eth-priv-to-addr';
+const dockerTag = 'latest';
+
+function promiseFromChildProcess(command: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    return exec(command, (err, stdout) => {
+      err ? reject(err) : resolve(stdout);
+    });
+  });
+}
+
+function getCleanPrivateKey(privKeyWallet: IFullWallet): string {
+  return stripHexPrefix(privKeyWallet.getPrivateKeyString());
+}
+
+function makeCommaSeparatedPrivateKeys(privKeyWallets: IFullWallet[]): string {
+  const privateKeys = privKeyWallets.map(getCleanPrivateKey);
+  return privateKeys.join(',');
+}
+
+async function privToAddrViaDocker(privKeyWallets: IFullWallet[]): Promise<string> {
+  const command = `docker run -e key=${makeCommaSeparatedPrivateKeys(
+    privKeyWallets
+  )} ${dockerImage}:${dockerTag}`;
+  const dockerOutput = await promiseFromChildProcess(command);
+  const newlineStrippedDockerOutput = dockerOutput.replace(/(\r\n|\n|\r)/gm, '');
+  return newlineStrippedDockerOutput;
+}
+
+function makeWallets(): IFullWallet[] {
+  const wallets: IFullWallet[] = [];
+  let i = 0;
+  while (i < derivationRounds) {
+    const privKeyWallet = generate();
+    wallets.push(privKeyWallet);
+    i += 1;
+  }
+  return wallets;
+}
+
+async function getNormalizedAddressFromWallet(wallet: IFullWallet): Promise<string> {
+  const privKeyWalletAddress = await wallet.getAddressString();
+  // strip checksum
+  return privKeyWalletAddress.toLowerCase();
+}
+
+async function getNormalizedAddressesFromWallets(wallets: IFullWallet[]): Promise<string[]> {
+  return Promise.all(wallets.map(getNormalizedAddressFromWallet));
+}
+
+async function testDerivation(): Promise<true> {
+  const wallets = makeWallets();
+  const walletAddrs = await getNormalizedAddressesFromWallets(wallets);
+  const dockerAddrsCS = await privToAddrViaDocker(wallets);
+  const dockerAddrs = dockerAddrsCS.split(',');
+  expect(walletAddrs).toEqual(dockerAddrs);
+  return true;
+}
+
+describe('Derivation Checker', () => {
+  beforeEach(() => {
+    // increase timer to prevent early timeout
+    jasmine.DEFAULT_TIMEOUT_INTERVAL = 100000;
+  });
+
+  it(`should derive identical addresses ${derivationRounds} times`, () => {
+    return testDerivation().then(expect);
+  });
+});