Running js tests in Electron renderer process (#1196)

* First take on running the tests inside an Electron render process

* Making progress

* Making it possible to override the location of the admin token

* Ignoreing the realm-object-server files

* Fixing a console.log to return a string instead of a boolean

But it should probably be removed entirely

* Making the downloaded ROS log less

* Adjusting comments in the test.sh

* Checking number of windows to determine an Electron crash

* Added a test that triggers garbage collection of ArrayBuffers

* Enabled the ability to run the tests in Electrons main process

* Run tests first in main process and the render

* Added a README.md that documents how Electron tests can be run

* Added a comment on why the garbage-collection test was added.

* Waiting with reading the admin token, until it is actually used

* Disabling the GarbageCollectionTests for now

scripts/download-object-server.sh

@@ -20,5 +20,7 @@ mkdir object-server-for-testing
 tar -C object-server-for-testing -xf "$object_server_bundle"
 rm "$object_server_bundle"
 
-echo -e "enterprise:\n  skip_setup: true\n" >> "object-server-for-testing/object-server/configuration.yml"
-touch "object-server-for-testing/object-server/do_not_open_browser"
+echo "enterprise:\n  skip_setup: true\n" >> "object-server-for-testing/object-server/configuration.yml"
+# Change to a "warn" level
+sed -i -- "s/# level: 'info'/level: 'warn'/g" object-server-for-testing/object-server/configuration.yml
+touch "object-server-for-testing/object-server/do_not_open_browser"

scripts/test.sh

@@ -355,6 +355,45 @@ case "$TARGET" in
   popd
   stop_server
   ;;
+"electron")
+  if [ "$(uname)" = 'Darwin' ]; then
+    download_server
+    start_server
+  fi
+
+  # Change to a temp directory - because this is what is done for node - but we pushd right after?
+  cd "$(mktemp -q -d -t realm.electron.XXXXXX)"
+  test_temp_dir=$PWD # set it to be cleaned at exit
+  pushd "$SRCROOT/tests/electron"
+
+  if [ "$(uname)" = 'Darwin' ]; then
+    npm install --build-from-source --realm_enable_sync
+  else
+    npm install --build-from-source
+  fi
+
+  # npm test -- --filter=ListTests
+  # npm test -- --filter=LinkingObjectsTests
+  # npm test -- --filter=ObjectTests
+  # npm test -- --filter=RealmTests
+  # npm test -- --filter=ResultsTests
+  # npm test -- --filter=QueryTests
+  # npm test -- --filter=MigrationTests
+  # npm test -- --filter=EncryptionTests
+  # npm test -- --filter=UserTests
+  # npm test -- --filter=SessionTests
+  # npm test -- --filter=GarbageCollectionTests
+  # npm test -- --filter=AsyncTests
+
+  npm test -- --process=main
+  npm test -- --process=render
+
+  popd
+
+  if [ "$(uname)" = 'Darwin' ]; then
+    stop_server
+  fi
+  ;;
 "test-runners")
   # Create a fake realm module that points to the source root so that test-runner tests can require('realm')
   npm install --build-from-source

tests/electron/.gitignore

@@ -0,0 +1 @@
+/realm-object-server

tests/electron/README.md

@@ -0,0 +1,40 @@
+# Realm JS tests running in an Electron enviroment
+
+Currently this directory consists of:
+- An electron app in `tests/electron/app` which has
+  - `jasmine.js` that imports the jasmin lib, setup a console logger and exports an execute function.
+  - `main.js` which starts a hidden `BrowserWindow` and either runs the tests itself (see --main flag below) or lets
+    the render process do the heavy lifting.
+  - `renderer.js` detects if it's supposed to run the tests and does that using the `jasmine.js`.
+- `spec.js` in which imports and executes the tests exported by `tests/js/index.js`.
+- A `test/electron/runner.js` script, which uses [spectron](https://www.npmjs.com/package/spectron) to start the Electron app and read out the console from the Electron process, and console logging it.
+
+## Flags
+
+To use these flags, you need to prepend them when calling `npm test` after the `--`, which indicates that the flag is
+not ment for npm.
+
+### Process
+
+You can specify in which Electron process to run the tests:
+- `--process=main` for the main process or
+- `--process=render` the render process (which is default)
+
+As an example, this runs all tests in the main process:
+
+    npm test -- --process=main
+
+### Filter
+
+If you want to run only a subset of the tests, use the `--filter` flag, ex:
+
+As an example, this runs only the suite named "UserTests":
+
+    npm test -- --filter=UserTests
+
+## Failing tests
+
+These tests are failing at the moment:
+- SessionTests (because REALM_MODULE_PATH is missing, due to `tests/spec/helpers` not loading correctly.
+- AsyncTests (because of the same reason as SessionTests)
+- GarbageCollectionTests (due to a bug that I'll be reporting soon)

tests/electron/app/index.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+  <script>
+    require('./renderer.js');
+  </script>
+</html>

tests/electron/app/jasmine.js

@@ -0,0 +1,26 @@
+"use strict";
+
+const Jasmine = require("jasmine");
+const JasmineConsoleReporter = require('jasmine-console-reporter');
+const path = require("path");
+
+const SPEC_PATH = path.join(__dirname, "..", "spec.js");
+
+const ADMIN_TOKEN_PATH = path.join(__dirname, "..", "..", "..", "object-server-for-testing", "admin_token.base64");
+process.env.ADMIN_TOKEN_PATH = ADMIN_TOKEN_PATH;
+
+// console.log(require.resolve("realm-spec-helpers"));
+exports.execute = (filter) => {
+  const jasmine = new Jasmine();
+
+  jasmine.clearReporters();
+  jasmine.addReporter(new JasmineConsoleReporter({
+    colors: 2,
+    cleanStack: 3,
+    verbosity: 4,
+    activity: false
+  }));
+  jasmine.execute([ SPEC_PATH ], filter);
+
+  return jasmine;
+};

tests/electron/app/main.js

@@ -0,0 +1,69 @@
+"use strict";
+
+// This file is pretty much a copy of https://github.com/electron/electron-quick-start/blob/master/main.js
+
+const electron = require("electron");
+// Module to control application life.
+const app = electron.app;
+// Increasing memory
+// app.commandLine.appendSwitch('js-flags', '--max-old-space-size=4096');
+// Module to create native browser window.
+const BrowserWindow = electron.BrowserWindow;
+
+const path = require("path");
+const url = require("url");
+
+const JASMIN_FILTER_KEY = "--filter";
+const MAIN_PROCESS_KEY = "--process";
+
+function getJasminFilter() {
+  const filterArg = process.argv.find((arg) => arg.indexOf(JASMIN_FILTER_KEY) === 0);
+  return filterArg ? filterArg.slice(JASMIN_FILTER_KEY.length + 1) : null;
+}
+
+function getProcess() {
+  const filterArg = process.argv.find((arg) => arg.indexOf(MAIN_PROCESS_KEY) === 0);
+  return filterArg ? filterArg.slice(MAIN_PROCESS_KEY.length + 1) : 'render';
+}
+
+const filter = getJasminFilter();
+const runIn = getProcess();
+
+// Keep a global reference of the window object, if you don´t, the window will
+// be closed automatically when the JavaScript object is garbage collected.
+let mainWindow;
+
+app.on("ready", () => {
+  // Create the browser window.
+  mainWindow = new BrowserWindow({
+    show: false
+  });
+
+  global.options = {
+    filter,
+    runIn
+  };
+
+  // Load the index.html of the app.
+  mainWindow.loadURL(url.format({
+    pathname: path.join(__dirname, "index.html"),
+    protocol: "file:",
+    slashes: true
+  }));
+
+  if (runIn === "main") {
+    console.log("Running tests in the main process.");
+    const jasmine = require("./jasmine.js").execute(filter);
+    jasmine.onComplete((passed) => {
+      process.exit(passed ? 0 : -1);
+    });
+  } else if(runIn === "render") {
+    console.log("Running tests in the render process.");
+  } else {
+    throw new Error("Can only run the tests in the 'main' or 'render' process");
+  }
+});
+
+app.on("quit", (e, exitCode) => {
+  console.log("Electron process stopped, with status", exitCode);
+});

tests/electron/app/renderer.js

@@ -0,0 +1,12 @@
+"use strict";
+
+const remote = require("electron").remote;
+
+const options = remote.getGlobal("options");
+if (options.runIn === "render") {
+  const jasmine = require("./jasmine.js").execute(options.filter);
+  jasmine.onComplete((passed) => {
+    // Add a delay if this happens too fast, to allow the WebDriver to connect first.
+    remote.process.exit(passed ? 0 : -1);
+  });
+}

tests/electron/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "realm-js-electron-tests",
+  "private": true,
+  "description": "Test harness running the Realm JS tests in the Electron renderer process.",
+  "main": "index.js",
+  "scripts": {
+    "prepublish": "electron-rebuild -t dev",
+    "start": "electron ./app/main.js",
+    "test": "node ./runner.js"
+  },
+  "devDependencies": {
+    "electron": "^1.6.11",
+    "electron-rebuild": "^1.6.0",
+    "jasmine": "../node_modules/jasmine",
+    "jasmine-console-reporter": "../node_modules/jasmine-console-reporter",
+    "realm": "../..",
+    "realm-tests": "../js",
+    "spectron": "^3.7.2"
+  }
+}

tests/electron/runner.js

@@ -0,0 +1,53 @@
+"use strict";
+
+const assert = require("assert");
+const path = require("path");
+const Application = require("spectron").Application;
+
+const ELECTRON_PATH = path.join(__dirname, "node_modules", ".bin", "electron");
+const MAIN_PATH = path.join(__dirname, "app", "main.js");
+const POLL_LOG_DELAY = 500;
+
+const filterOption = process.argv[2] || null;
+
+const doneMatcher = /Electron process stopped, with status ([-\d]+)/;
+
+const app = new Application({
+  path: ELECTRON_PATH,
+  args: [ MAIN_PATH ].concat(process.argv.slice(2))
+});
+
+console.log("Trying to start an Electron process.");
+
+app.start().then(() => {
+  console.log("The following messages are logs from the Electron process:");
+  // Keep reading the log, until Jasmine prints "ALL DONE"
+  return new Promise((resolve, reject) => {
+    const timeout = setInterval(() => {
+      app.client.getMainProcessLogs().then((logs) => {
+        logs.forEach((msg) => {
+          console.log(msg);
+          const doneTest = doneMatcher.exec(msg);
+          if(doneTest) {
+            const statusCode = parseInt(doneTest[1], 10);
+            clearTimeout(timeout);
+            resolve(statusCode);
+          }
+        });
+        app.client.getWindowCount().then((count) => {
+          if(count === 0) {
+            const err = new Error("All Electron windows unexpectedly closed.");
+            reject(err);
+          }
+        });
+      });
+    }, POLL_LOG_DELAY);
+  });
+}).then((statusCode) => {
+  // Exit with the same status as the Electron process
+  process.exit(statusCode);
+}).catch((error) => {
+  // Log any failures
+  console.error("Test harness failure:", error.message);
+  process.exit(-1);
+})

tests/electron/spec.js

@@ -0,0 +1,95 @@
+"use strict";
+
+const assert = require("assert");
+const path = require("path");
+const fs = require("fs");
+
+const Realm = require("realm");
+const RealmTests = require("realm-tests");
+
+describe("Test harness", () => {
+  if(global.options && global.options.runIn === "main") {
+    it("runs the test in the main process", () => {
+      assert(process.versions.chrome, "Expected a chrome version");
+      assert(!global.window, "Expected no window constant");
+      assert(!global.navigator, "Expected no navigator global");
+    });
+  } else {
+    it("runs the test in the browser process", () => {
+      assert(process.versions.chrome, "Expected a chrome version");
+      assert(global.window, "Expected a window constant");
+
+      const userAgent = global.navigator.userAgent;
+      assert(userAgent.indexOf("Electron") >= 0, "Expected Electron in the user-agent");
+      assert(userAgent.indexOf("Chrome") >= 0, "Expected Chrome in the user-agent");
+    });
+  }
+
+  it("waits for async tests to complete", (done) => {
+    setTimeout(() => {
+      done();
+    }, 1000);
+  });
+
+  it("loads Realm", () => {
+    assert(Realm);
+    assert.equal(typeof(Realm), "function");
+    assert.equal(Realm.name, "Realm");
+  });
+
+  /*
+  it("fails", (done) => {
+    assert(false);
+  });
+  */
+});
+
+// Almost a copy-paste from the ../spec/unit_tests.js - so it might be possible to generalize.
+
+// Setting the timeout to the same as the ../../spec/unit_tests.js
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 30000;
+
+Realm.copyBundledRealmFiles = function() {
+  const sourceDir = path.join(__dirname, '../data');
+  const destinationDir = path.dirname(Realm.defaultPath);
+
+  for (let filename of fs.readdirSync(sourceDir)) {
+    let src = path.join(sourceDir, filename);
+    let dest = path.join(destinationDir, filename);
+
+    // If the destination file already exists, then don't overwrite it.
+    try {
+        fs.accessSync(dest);
+        continue;
+    } catch (e) {}
+
+    fs.writeFileSync(dest, fs.readFileSync(src));
+  }
+};
+
+const tests = RealmTests.getTestNames();
+for (const suiteName in tests) {
+  describe(suiteName, () => {
+
+    beforeAll(done => RealmTests.prepare(done));
+
+    beforeEach(() => RealmTests.runTest(suiteName, 'beforeEach'));
+
+    for (const testName of tests[suiteName]) {
+      it(testName, (done) => {
+        try {
+          let result = RealmTests.runTest(suiteName, testName);
+          if (result instanceof Promise) {
+            result.then(done, done.fail.bind(done));
+          } else {
+            done();
+          }
+        } catch (e) {
+          done.fail(e);
+        }
+      });
+    }
+
+    afterEach(() => RealmTests.runTest(suiteName, 'afterEach'));
+  });
+}

tests/js/admin-user-helper.js

@@ -1,12 +1,23 @@
 'use strict';
-function node_require(module) { 
-    return require(module); 
+function node_require(module) {
+    return require(module);
 
 }
 let fs = node_require("fs");
 let path = node_require("path");
 var Realm = node_require('realm');
 
+const DEFAULT_ADMIN_TOKEN_PATH = path.join(__dirname, "..", "..", "object-server-for-testing", "admin_token.base64");
+const ADMIN_TOKEN_PATH = process.env.ADMIN_TOKEN_PATH || DEFAULT_ADMIN_TOKEN_PATH;
+
+function getAdminToken() {
+  if(fs.existsSync(ADMIN_TOKEN_PATH)) {
+    return fs.readFileSync(ADMIN_TOKEN_PATH, 'utf-8');
+  } else {
+    throw new Error("Missing the file with an admin token: " + ADMIN_TOKEN_PATH);
+  }
+}
+
 function random(min, max) {
     min = Math.ceil(min);
     max = Math.floor(max);
@@ -21,11 +32,11 @@ exports.createAdminUser = function () {
         Realm.Sync.User.register('http://localhost:9080', newAdminName, password, (error, user) => {
             if (error) {
                 reject(error);
-            } else {  
+            } else {
                 let userIdentity = user.identity;
                 user.logout();
 
-                let admin_token_user = Realm.Sync.User.adminUser(fs.readFileSync(path.join(__dirname, '/../../object-server-for-testing/admin_token.base64'), 'utf-8'));                
+                let admin_token_user = Realm.Sync.User.adminUser(getAdminToken());
 
                 const config = {
                     sync: {
@@ -62,8 +73,8 @@ exports.createAdminUser = function () {
                                     return;
                                 }
 
-                                resolve({ 
-                                    username: newAdminName, 
+                                resolve({
+                                    username: newAdminName,
                                     password
                                 });
                             }
@@ -76,4 +87,3 @@ exports.createAdminUser = function () {
         });
     });
 }
-

tests/js/garbage-collection.js

@@ -0,0 +1,70 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2016 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+/*
+ * This test suite is trying to make a bunch of objects with properties of type "data", which will use an ArrayBuffer
+ * when accessed.
+ * The reason for this test suite is that we have experianced issues in the current version (v1.6.11) of Electron that
+ * will crash when an v8::ArrayBuffer is garbage collected.
+ * @see https://github.com/electron/electron/issues/2601#issuecomment-135258750
+ */
+
+'use strict';
+
+const Realm = require('realm');
+const TestCase = require('./asserts');
+
+const NUMBER_OF_OBJECTS = 1000;
+const BUFFER_LENGTH = 1024;
+const READ_CYCLES = 10;
+
+module.exports = {
+  testPropertiesOfData: () => {
+
+    const TestingSchema = {
+      name: 'Testing',
+      properties: {
+        n: 'int',
+        someData: 'data'
+      }
+    };
+
+    // Create a new realm
+    const realm = new Realm({schema: [TestingSchema]});
+    // Add a bunch of objects, with "data" to it
+    realm.write(() => {
+      for(let i = 0; i < NUMBER_OF_OBJECTS; i++) {
+        realm.create('Testing', {
+          n: i,
+          someData: new ArrayBuffer(BUFFER_LENGTH),
+        });
+      }
+    });
+
+    for (let readCycle = 0; readCycle < READ_CYCLES; readCycle++) {
+      let allObjects = realm.objects('Testing');
+      let totalBytes = 0;
+      for (let object of allObjects) {
+        let toBeFreed = object.someData;
+        // Accessing the byteLength of the objects someData property
+        totalBytes += toBeFreed.byteLength;
+      }
+      // console.log(`Read a total of ${totalBytes} bytes.`);
+    }
+  }
+};

tests/js/index.js

@@ -27,7 +27,8 @@ var TESTS = {
     RealmTests: require('./realm-tests'),
     ResultsTests: require('./results-tests'),
     QueryTests: require('./query-tests'),
-    MigrationTests: require('./migration-tests')
+    MigrationTests: require('./migration-tests'),
+    // GarbageCollectionTests: require('./garbage-collection'),
 };
 
 // encryption is not supported on windows

tests/js/session-tests.js

@@ -26,7 +26,7 @@ const Realm = require('realm');
 const TestCase = require('./asserts');
 
 const isNodeProccess = (typeof process === 'object' && process + '' === '[object process]');
-console.log("isnode " + isNodeProccess + " typeof " + typeof process === 'object');
+console.log("isnode " + isNodeProccess + " typeof " + (typeof(process) === 'object'));
 function node_require(module) {
     return require(module);
 }