[react-packager] Introduce `require.ensure`

Summary:
This is the first step to add support for splitting the JS bundle into multiple ones. This diff adds support for keeping track of the async dependencies each module has. To do so we introduce the following syntax:

  require.ensure(['dep1', 'dep2, ..., 'depN'], callback);

Where the callback function is asynchronously invoked once all the indicated modules are loaded.

Internally, the packager keeps track of every set of async dependencies a module has. So for instance if a module looks like this:
  require.ensure(['dep1'], () => {...});
  require.ensure(['dep2'], () => {...});

the `Module` object will keep track of each set of dependencies separately (because we might want to put them on separate bundles).

react-packager/src/DependencyResolver/AssetModule.js

@@ -14,6 +14,10 @@ class AssetModule extends Module {
     return Promise.resolve([]);
   }
 
+  getAsyncDependencies() {
+    return Promise.resolve([]);
+  }
+
   _read() {
     return Promise.resolve({});
   }

react-packager/src/DependencyResolver/AssetModule_DEPRECATED.js

@@ -17,6 +17,10 @@ class AssetModule_DEPRECATED extends Module {
     return Promise.resolve([]);
   }
 
+  getAsyncDependencies() {
+    return Promise.resolve([]);
+  }
+
   getPlainObject() {
     const {name, resolution} = getAssetDataFromName(this.path);
 

react-packager/src/DependencyResolver/Module.js

@@ -1,3 +1,11 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 'use strict';
 
 const Promise = require('promise');
@@ -69,6 +77,10 @@ class Module {
     this._cache.invalidate(this.path);
   }
 
+  getAsyncDependencies() {
+    return this._read().then(data => data.asyncDependencies);
+  }
+
   _read() {
     if (!this._reading) {
       this._reading = this._fastfs.readFile(this.path).then(content => {
@@ -85,7 +97,9 @@ class Module {
         if ('extern' in moduleDocBlock) {
           data.dependencies = [];
         } else {
-          data.dependencies = extractRequires(content);
+          var dependencies = extractRequires(content);
+          data.dependencies = dependencies.sync;
+          data.asyncDependencies = dependencies.async;
         }
 
         return data;
@@ -124,20 +138,68 @@ class Module {
 /**
  * Extract all required modules from a `code` string.
  */
-var blockCommentRe = /\/\*(.|\n)*?\*\//g;
-var lineCommentRe = /\/\/.+(\n|$)/g;
+const blockCommentRe = /\/\*(.|\n)*?\*\//g;
+const lineCommentRe = /\/\/.+(\n|$)/g;
+const trailingCommaRe = /,\s*$/g;
+const removeSpacesRe = /\s/g;
+const quotesRe = /'/g;
 function extractRequires(code /*: string*/) /*: Array<string>*/ {
-  var deps = [];
+  var deps = {
+    sync: [],
+    async: [],
+  };
 
   code
     .replace(blockCommentRe, '')
     .replace(lineCommentRe, '')
+    // Parse sync dependencies. See comment below for further detils.
     .replace(replacePatterns.IMPORT_RE, (match, pre, quot, dep, post) => {
-      deps.push(dep);
+      deps.sync.push(dep);
       return match;
     })
-    .replace(replacePatterns.REQUIRE_RE, function(match, pre, quot, dep, post) {
-      deps.push(dep);
+    // Parse the sync dependencies this module has. When the module is
+    // required, all it's sync dependencies will be loaded into memory.
+    // Sync dependencies can be defined either using `require` or the ES6
+    // `import` syntax:
+    //   var dep1 = require('dep1');
+    .replace(replacePatterns.REQUIRE_RE, (match, pre, quot, dep, post) => {
+      deps.sync.push(dep);
+    })
+    // Parse async dependencies this module has. As opposed to what happens
+    // with sync dependencies, when the module is required, it's async
+    // dependencies won't be loaded into memory. This is deferred till the
+    // code path gets to a `require.ensure` statement. The syntax is similar
+    // to webpack's one:
+    //   require.ensure(['dep1', 'dep2'], () => {
+    //     var dep1 = require('dep1');
+    //     var dep2 = require('dep2');
+    //     // do something with dep1 and dep2
+    //   });
+    .replace(replacePatterns.REQUIRE_ENSURE_RE, (match, dep, post) => {
+      dep = dep
+        .replace(blockCommentRe, '')
+        .replace(lineCommentRe, '')
+        .replace(trailingCommaRe, '')
+        .replace(removeSpacesRe, '')
+        .replace(quotesRe, '"');
+
+      if (dep) {
+        try {
+          dep = JSON.parse('[' + dep + ']');
+        } catch(e) {
+          throw 'Error processing `require.ensure` while attemping to parse ' +
+                'dependencies `[' + dep + ']`: ' + e;
+        }
+
+        dep.forEach(d => {
+          if (typeof d !== 'string') {
+            throw 'Error processing `require.ensure`: dependencies `[' +
+                  d + ']` must be string literals';
+          }
+        });
+
+        deps.async.push(dep);
+      }
     });
 
   return deps;

react-packager/src/DependencyResolver/__tests__/HasteDependencyResolver-test.js

@@ -479,8 +479,8 @@ describe('HasteDependencyResolver', function() {
       }, code).then(processedCode => {
 
         expect(processedCode).toEqual([
-          '__d(\'test module\',["changed","Y"],function(global,' +
-            ' require, requireDynamic, requireLazy, module, exports) {  ' +
+          '__d(\'test module\',["changed","Y"],function(global, require,' +
+            ' module, exports) {  ' +
             "import'x';",
           "import 'changed';",
           "import 'changed' ;",

react-packager/src/DependencyResolver/__tests__/Module-test.js

@@ -0,0 +1,133 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+jest
+  .dontMock('absolute-path')
+  .dontMock('../fastfs')
+  .dontMock('../replacePatterns')
+  .dontMock('../DependencyGraph/docblock')
+  .dontMock('../../FileWatcher')
+  .dontMock('../Module');
+
+jest
+  .mock('fs');
+
+describe('Module', () => {
+  var Fastfs;
+  var Module;
+  var ModuleCache;
+  var Promise;
+  var fs;
+
+  const FileWatcher = require('../../FileWatcher');
+  const fileWatcher = new FileWatcher(['/root']);
+
+  beforeEach(function() {
+    Fastfs = require('../fastfs');
+    Module = require('../Module');
+    ModuleCache = require('../ModuleCache');
+    Promise = require('promise');
+    fs = require('fs');
+  });
+
+  describe('Async Dependencies', () => {
+    function expectAsyncDependenciesToEqual(expected) {
+      var fastfs = new Fastfs(
+        ['/root'],
+        fileWatcher,
+        {crawling: Promise.resolve(['/root/index.js']), ignore: []},
+      );
+
+      return fastfs.build().then(() => {
+        var module = new Module('/root/index.js', fastfs, new ModuleCache(fastfs));
+
+        return module.getAsyncDependencies().then(actual =>
+          expect(actual).toEqual(expected)
+        );
+      });
+    }
+
+    pit('should recognize single dependency', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure(["dep1"], function() {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1']]);
+    });
+
+    pit('should parse single quoted dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure([\'dep1\'], function() {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1']]);
+    });
+
+    pit('should recognize multiple dependencies on the same statement', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure(["dep1", "dep2"], function() {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1', 'dep2']]);
+    });
+
+    pit('should group async dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': [
+            'require.ensure(["dep1", "dep2"], function() {});',
+            'require.ensure(["dep3", "dep4"], function() {});',
+          ].join('\n'),
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([
+        ['dep1', 'dep2'],
+        ['dep3', 'dep4']
+      ]);
+    });
+
+    pit('shouldn\'t throw with ES6 arrow functions', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure(["dep1", "dep2"], () => {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1', 'dep2']]);
+    });
+
+    pit('parse fine new lines', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure(["dep1", \n"dep2"], () => {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1', 'dep2']]);
+    });
+
+    pit('ignore comments', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': 'require.ensure(["dep1", /*comment*/"dep2"], () => {});',
+        }
+      });
+
+      return expectAsyncDependenciesToEqual([['dep1', 'dep2']]);
+    });
+  });
+});

react-packager/src/DependencyResolver/index.js

@@ -176,8 +176,7 @@ function defineModuleCode({moduleName, code, deps}) {
     `__d(`,
     `'${moduleName}',`,
     `${deps},`,
-    'function(global, require, ',
-    'requireDynamic, requireLazy, module, exports) {',
+    'function(global, require, module, exports) {',
     `  ${code}`,
     '\n});',
   ].join('');

react-packager/src/DependencyResolver/polyfills/require.js

@@ -303,6 +303,18 @@
     return _totalFactories;
   };
 
+  /**
+   * Asynchronously loads any missing dependency and executes the provided
+   * callback once all of them are satisfied.
+   *
+   * Note that the dependencies on the provided array must be string literals
+   * as the packager uses this information to figure out how the modules are
+   * packaged into different bundles.
+   */
+  require.ensure = function(dependencies, callback) {
+    throw '`require.ensure` is still not supported';
+  };
+
   /**
   * The define function conforming to CommonJS proposal:
   * http://wiki.commonjs.org/wiki/Modules/AsynchronousDefinition
@@ -464,56 +476,6 @@
     }
   }
 
-  /**
-   * Special version of define that executes the factory as soon as all
-   * dependencies are met.
-   *
-   * define() does just that, defines a module. Module's factory will not be
-   * called until required by other module. This makes sense for most of our
-   * library modules: we do not want to execute the factory unless it's being
-   * used by someone.
-   *
-   * On the other hand there are modules, that you can call "entrance points".
-   * You want to run the "factory" method for them as soon as all dependencies
-   * are met.
-   *
-   * @example
-   *
-   *   define('BaseClass', [], function() { return ... });
-   *   // ^^ factory for BaseClass was just stored in modulesMap
-   *
-   *   define('SubClass', ['BaseClass'], function() { ... });
-   *   // SubClass module is marked as ready (waiting == 0), factory is just
-   *   // stored
-   *
-   *   define('OtherClass, ['BaseClass'], function() { ... });
-   *   // OtherClass module is marked as ready (waiting == 0), factory is just
-   *   // stored
-   *
-   *   requireLazy(['SubClass', 'ChatConfig'],
-   *     function() { ... });
-   *   // ChatRunner is waiting for ChatConfig to come
-   *
-   *   define('ChatConfig', [], { foo: 'bar' });
-   *   // at this point ChatRunner is marked as ready, and its factory
-   *   // executed + all dependent factories are executed too: BaseClass,
-   *   // SubClass, ChatConfig notice that OtherClass's factory won't be
-   *   // executed unless explicitly required by someone
-   *
-   * @param {Array} dependencies
-   * @param {Object|Function} factory
-   */
-  function requireLazy(dependencies, factory, context) {
-    return define(
-      dependencies,
-      factory,
-      undefined,
-      REQUIRE_WHEN_READY,
-      context,
-      1
-    );
-  }
-
   function _uid() {
     return '__mod__' + _counter++;
   }
@@ -595,12 +557,8 @@
 
   _register('global', global);
   _register('require', require);
-  _register('requireDynamic', require);
-  _register('requireLazy', requireLazy);
 
   global.require = require;
-  global.requireDynamic = require;
-  global.requireLazy = requireLazy;
 
   require.__debug = {
     modules: modulesMap,
@@ -621,8 +579,7 @@
    * out for every module which would be a lot of extra bytes.
    */
   global.__d = function(id, deps, factory, _special, _inlineRequires) {
-    var defaultDeps = ['global', 'require', 'requireDynamic', 'requireLazy',
-                       'module', 'exports'];
+    var defaultDeps = ['global', 'require', 'module', 'exports'];
     define(id, defaultDeps.concat(deps), factory, _special || USED_AS_TRANSPORT,
            null, null, _inlineRequires);
   };

react-packager/src/DependencyResolver/replacePatterns.js

@@ -11,3 +11,4 @@
 
 exports.IMPORT_RE = /(\bimport\s+?(?:.+\s+?from\s+?)?)(['"])([^'"]+)(\2)/g;
 exports.REQUIRE_RE = /(\brequire\s*?\(\s*?)(['"])([^'"]+)(\2\s*?\))/g;
+exports.REQUIRE_ENSURE_RE = /\brequire\.ensure\s*\(\s*(?:\[([^\]]+)\])?/g;