[react-packager] Introduce Bundler

Summary:
Introduce a Bundler capable of generating the layout of modules for a given entry point. The current algorithm is the most trivial we could come up with: (1)it puts all the sync dependencies into the same bundle and (2) each group of async  dependencies with all their dependencies into a separate bundle. For async dependencies we do this recursivelly, meaning that async dependencies could have async dependencies which will end up on separate bundles as well.

The output of of the layout is an array of bundles. Each bundle is just an array for now with the dependencies in the order the requires where processed. Using this information we should be able to generate the actual bundles by using the `/path/to/entry/point.bundle` endpoint. We might change the structure of this json in the future, for instance to account for parent/child bundles relationships.

The next step will be to improve this algorithm to avoid repeating quite a bit dependencies across bundles.

packager/react-packager/src/BundlesLayout/__tests__/BundlesLayout-test.js

@@ -0,0 +1,150 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+jest
+  .dontMock('../index');
+
+const Promise = require('promise');
+
+describe('BundlesLayout', () => {
+  var BundlesLayout;
+  var DependencyResolver;
+
+  beforeEach(() => {
+    BundlesLayout = require('../index');
+    DependencyResolver = require('../../DependencyResolver');
+  });
+
+  describe('generate', () => {
+    function newBundlesLayout() {
+      return new BundlesLayout({
+        dependencyResolver: new DependencyResolver(),
+      });
+    }
+
+    function dep(path) {
+      return {path};
+    }
+
+    pit('should bundle sync dependencies', () => {
+      DependencyResolver.prototype.getDependencies.mockImpl((path) => {
+        switch (path) {
+          case '/root/index.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/index.js'), dep('/root/a.js')],
+              asyncDependencies: [],
+            });
+          case '/root/a.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/a.js')],
+              asyncDependencies: [],
+            });
+          default:
+            throw 'Undefined path: ' + path;
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(bundles).toEqual([
+          [dep('/root/index.js'), dep('/root/a.js')],
+        ])
+      );
+    });
+
+    pit('should separate async dependencies into different bundle', () => {
+      DependencyResolver.prototype.getDependencies.mockImpl((path) => {
+        switch (path) {
+          case '/root/index.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/index.js')],
+              asyncDependencies: [['/root/a.js']],
+            });
+          case '/root/a.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/a.js')],
+              asyncDependencies: [],
+            });
+          default:
+            throw 'Undefined path: ' + path;
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(bundles).toEqual([
+          [dep('/root/index.js')],
+          [dep('/root/a.js')],
+        ])
+      );
+    });
+
+    pit('separate async dependencies of async dependencies', () => {
+      DependencyResolver.prototype.getDependencies.mockImpl((path) => {
+        switch (path) {
+          case '/root/index.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/index.js')],
+              asyncDependencies: [['/root/a.js']],
+            });
+          case '/root/a.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/a.js')],
+              asyncDependencies: [['/root/b.js']],
+            });
+          case '/root/b.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/b.js')],
+              asyncDependencies: [],
+            });
+          default:
+            throw 'Undefined path: ' + path;
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(bundles).toEqual([
+          [dep('/root/index.js')],
+          [dep('/root/a.js')],
+          [dep('/root/b.js')],
+        ])
+      );
+    });
+
+    pit('separate bundle sync dependencies of async ones on same bundle', () => {
+      DependencyResolver.prototype.getDependencies.mockImpl((path) => {
+        switch (path) {
+          case '/root/index.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/index.js')],
+              asyncDependencies: [['/root/a.js']],
+            });
+          case '/root/a.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/a.js'), dep('/root/b.js')],
+              asyncDependencies: [],
+            });
+          case '/root/b.js':
+            return Promise.resolve({
+              dependencies: [dep('/root/b.js')],
+              asyncDependencies: [],
+            });
+          default:
+            throw 'Undefined path: ' + path;
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(bundles).toEqual([
+          [dep('/root/index.js')],
+          [dep('/root/a.js'), dep('/root/b.js')],
+        ])
+      );
+    });
+  });
+});

packager/react-packager/src/BundlesLayout/__tests__/BundlesLayoutIntegration-test.js

@@ -0,0 +1,512 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+jest
+  .dontMock('absolute-path')
+  .dontMock('crypto')
+  .dontMock('underscore')
+  .dontMock('../index')
+  .dontMock('../../lib/getAssetDataFromName')
+  .dontMock('../../DependencyResolver/crawlers')
+  .dontMock('../../DependencyResolver/crawlers/node')
+  .dontMock('../../DependencyResolver/DependencyGraph/docblock')
+  .dontMock('../../DependencyResolver/fastfs')
+  .dontMock('../../DependencyResolver/replacePatterns')
+  .dontMock('../../DependencyResolver')
+  .dontMock('../../DependencyResolver/DependencyGraph')
+  .dontMock('../../DependencyResolver/AssetModule_DEPRECATED')
+  .dontMock('../../DependencyResolver/AssetModule')
+  .dontMock('../../DependencyResolver/Module')
+  .dontMock('../../DependencyResolver/Package')
+  .dontMock('../../DependencyResolver/ModuleCache');
+
+const Promise = require('promise');
+
+jest.mock('fs');
+
+describe('BundlesLayout', () => {
+  var BundlesLayout;
+  var Cache;
+  var DependencyResolver;
+  var fileWatcher;
+  var fs;
+
+  beforeEach(() => {
+    fs = require('fs');
+    BundlesLayout = require('../index');
+    Cache = require('../../Cache');
+    DependencyResolver = require('../../DependencyResolver');
+
+    fileWatcher = {
+      on: () => this,
+      isWatchman: () => Promise.resolve(false)
+    };
+  });
+
+  describe('generate', () => {
+    const polyfills = [
+      'polyfills/prelude_dev.js',
+      'polyfills/prelude.js',
+      'polyfills/require.js',
+      'polyfills/polyfills.js',
+      'polyfills/console.js',
+      'polyfills/error-guard.js',
+      'polyfills/String.prototype.es6.js',
+      'polyfills/Array.prototype.es6.js',
+    ];
+
+    function newBundlesLayout() {
+      const resolver = new DependencyResolver({
+        projectRoots: ['/root'],
+        fileWatcher: fileWatcher,
+        cache: new Cache(),
+        assetExts: ['js', 'png'],
+        assetRoots: ['/root'],
+      });
+
+      return new BundlesLayout({dependencyResolver: resolver});
+    }
+
+    function modulePaths(bundles) {
+      if (!bundles) {
+        return null;
+      }
+
+      return bundles.map(bundle => {
+        return bundle
+          .filter(module => { // filter polyfills
+            for (let p of polyfills) {
+              if (module.id.indexOf(p) !== -1) {
+                return false;
+              }
+            }
+            return true;
+          })
+          .map(module => module.path);
+      });
+    }
+
+    pit('should bundle dependant modules', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require("a");`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js', '/root/a.js'],
+        ])
+      );
+    });
+
+    pit('should split bundles for async dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js'],
+        ])
+      );
+    });
+
+    pit('should split into multiple bundles separate async dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);
+            require.ensure(["b"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js'],
+          ['/root/b.js'],
+        ])
+      );
+    });
+
+    pit('should put related async dependencies into the same bundle', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a", "b"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/b.js'],
+        ])
+      );
+    });
+
+    pit('should fully traverse sync dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require("a");
+            require.ensure(["b"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js', '/root/a.js'],
+          ['/root/b.js'],
+        ])
+      );
+    });
+
+    pit('should include sync dependencies async dependencies might have', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */,
+            require("b");`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */
+            require("c");`,
+          'c.js': `
+            /**
+             * @providesModule c
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/b.js', '/root/c.js'],
+        ])
+      );
+    });
+
+    pit('should allow duplicated dependencies across bundles', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);
+            require.ensure(["b"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */,
+            require("c");`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */
+            require("c");`,
+          'c.js': `
+            /**
+             * @providesModule c
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/c.js'],
+          ['/root/b.js', '/root/c.js'],
+        ])
+      );
+    });
+
+    pit('should put in separate bundles async dependencies of async dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */,
+            require.ensure(["b"]);`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */
+            require("c");`,
+          'c.js': `
+            /**
+             * @providesModule c
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js'],
+          ['/root/b.js', '/root/c.js'],
+        ])
+      );
+    });
+
+    pit('should dedup same async bundle duplicated dependencies', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a", "b"]);`,
+          'a.js': `
+            /**,
+             * @providesModule a
+             */,
+            require("c");`,
+          'b.js': `
+            /**
+             * @providesModule b
+             */
+            require("c");`,
+          'c.js': `
+            /**
+             * @providesModule c
+             */`,
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/c.js', '/root/b.js'],
+        ])
+      );
+    });
+
+    pit('should put image dependencies into separate bundles', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);`,
+          'a.js':`
+            /**,
+             * @providesModule a
+             */,
+            require("./img.png");`,
+          'img.png': '',
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/img.png'],
+        ])
+      );
+    });
+
+    pit('should put image dependencies across bundles', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);
+            require.ensure(["b"]);`,
+          'a.js':`
+            /**,
+             * @providesModule a
+             */,
+            require("./img.png");`,
+          'b.js':`
+            /**,
+             * @providesModule b
+             */,
+            require("./img.png");`,
+          'img.png': '',
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/img.png'],
+          ['/root/b.js', '/root/img.png'],
+        ])
+      );
+    });
+
+    pit('could async require asset', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["./img.png"]);`,
+          'img.png': '',
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/img.png'],
+        ])
+      );
+    });
+
+    pit('should include deprecated assets into separate bundles', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["a"]);`,
+          'a.js':`
+            /**,
+             * @providesModule a
+             */,
+            require("image!img");`,
+          'img.png': '',
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/a.js', '/root/img.png'],
+        ])
+      );
+    });
+
+    pit('could async require deprecated asset', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["image!img"]);`,
+          'img.png': '',
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/img.png'],
+        ])
+      );
+    });
+
+    pit('should put packages into bundles', () => {
+      fs.__setMockFilesystem({
+        'root': {
+          'index.js': `
+            /**
+             * @providesModule index
+             */
+            require.ensure(["aPackage"]);`,
+          'aPackage': {
+            'package.json': JSON.stringify({
+              name: 'aPackage',
+              main: './main.js',
+              browser: {
+                './main.js': './client.js',
+              },
+            }),
+            'main.js': 'some other code',
+            'client.js': 'some code',
+          },
+        }
+      });
+
+      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        expect(modulePaths(bundles)).toEqual([
+          ['/root/index.js'],
+          ['/root/aPackage/client.js'],
+        ])
+      );
+    });
+  });
+});

packager/react-packager/src/BundlesLayout/index.js

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const _ = require('underscore');
+const declareOpts = require('../lib/declareOpts');
+
+const validateOpts = declareOpts({
+  dependencyResolver: {
+    type: 'object',
+    required: true,
+  },
+});
+
+/**
+ * Class that takes care of separating the graph of dependencies into
+ * separate bundles
+ */
+class BundlesLayout {
+  constructor(options) {
+    const opts = validateOpts(options);
+    this._resolver = opts.dependencyResolver;
+  }
+
+  generateLayout(entryPaths, isDev) {
+    const bundles = [];
+    var pending = [entryPaths];
+
+    return promiseWhile(
+      () => pending.length > 0,
+      () => bundles,
+      () => {
+        const pendingPaths = pending.shift();
+        return Promise
+          .all(pendingPaths.map(path =>
+            this._resolver.getDependencies(path, {dev: isDev})
+          ))
+          .then(modulesDeps => {
+            let syncDependencies = Object.create(null);
+            modulesDeps.forEach(moduleDeps => {
+              moduleDeps.dependencies.forEach(dep =>
+                syncDependencies[dep.path] = dep
+              );
+              pending = pending.concat(moduleDeps.asyncDependencies);
+            });
+
+            syncDependencies = _.values(syncDependencies);
+            if (syncDependencies.length > 0) {
+              bundles.push(syncDependencies);
+            }
+
+            return Promise.resolve(bundles);
+          });
+      },
+    );
+  }
+}
+
+// Runs the body Promise meanwhile the condition callback is satisfied.
+// Once it's not satisfied anymore, it returns what the results callback
+// indicates
+function promiseWhile(condition, result, body) {
+  if (!condition()) {
+    return Promise.resolve(result());
+  }
+
+  return body().then(() => promiseWhile(condition, result, body));
+}
+
+module.exports = BundlesLayout;

packager/react-packager/src/DependencyResolver/DependencyGraph/index.js

@@ -162,33 +162,7 @@ class DependencyGraph {
 
   getOrderedDependencies(entryPath) {
     return this.load().then(() => {
-      const absPath = this._getAbsolutePath(entryPath);
+      const entry = this._getModuleForEntryPath(entryPath);
-
-      if (absPath == null) {
-        throw new NotFoundError(
-          'Could not find source file at %s',
-          entryPath
-        );
-      }
-
-      const absolutePath = path.resolve(absPath);
-
-      if (absolutePath == null) {
-        throw new NotFoundError(
-          'Cannot find entry file %s in any of the roots: %j',
-          entryPath,
-          this._opts.roots
-        );
-      }
-
-      const platformExt = getPlatformExt(entryPath);
-      if (platformExt && this._opts.platforms.indexOf(platformExt) > -1) {
-        this._platformExt = platformExt;
-      } else {
-        this._platformExt = null;
-      }
-
-      const entry = this._moduleCache.getModule(absolutePath);
       const deps = [];
       const visited = Object.create(null);
       visited[entry.hash()] = true;
@@ -225,7 +199,23 @@ class DependencyGraph {
       };
 
       return collect(entry)
-        .then(() => Promise.all(deps.map(dep => dep.getPlainObject())));
+        .then(() => Promise.all(deps.map(dep => dep.getPlainObject())))
+        .then();
+    });
+  }
+
+  getAsyncDependencies(entryPath) {
+    return this.load().then(() => {
+      const mod = this._getModuleForEntryPath(entryPath);
+      return mod.getAsyncDependencies().then(bundles =>
+        Promise
+          .all(bundles.map(bundle =>
+            Promise.all(bundle.map(
+              dep => this.resolveDependency(mod, dep)
+            ))
+          ))
+          .then(bs => bs.map(bundle => bundle.map(dep => dep.path)))
+      );
     });
   }
 
@@ -245,6 +235,36 @@ class DependencyGraph {
     return null;
   }
 
+  _getModuleForEntryPath(entryPath) {
+    const absPath = this._getAbsolutePath(entryPath);
+
+    if (absPath == null) {
+      throw new NotFoundError(
+        'Could not find source file at %s',
+        entryPath
+      );
+    }
+
+    const absolutePath = path.resolve(absPath);
+
+    if (absolutePath == null) {
+      throw new NotFoundError(
+        'Cannot find entry file %s in any of the roots: %j',
+        entryPath,
+        this._opts.roots
+      );
+    }
+
+    const platformExt = getPlatformExt(entryPath);
+    if (platformExt && this._opts.platforms.indexOf(platformExt) > -1) {
+      this._platformExt = platformExt;
+    } else {
+      this._platformExt = null;
+    }
+
+    return this._moduleCache.getModule(absolutePath);
+  }
+
   _resolveHasteDependency(fromModule, toModuleName) {
     toModuleName = normalizePath(toModuleName);
 

packager/react-packager/src/DependencyResolver/Module.js

@@ -198,6 +198,8 @@ function extractRequires(code /*: string*/) /*: Array<string>*/ {
           }
         });
 
+        // TODO: throw error if there are duplicate dependencies
+
         deps.async.push(dep);
       }
     });

packager/react-packager/src/DependencyResolver/index.js

@@ -83,22 +83,26 @@ HasteDependencyResolver.prototype.getDependencies = function(main, options) {
 
   var depGraph = this._depGraph;
   var self = this;
-  return depGraph.load().then(
+  return depGraph
-    () => depGraph.getOrderedDependencies(main).then(
+    .load()
-      dependencies => {
+    .then(() => Promise.all([
+      depGraph.getOrderedDependencies(main),
+      depGraph.getAsyncDependencies(main),
+    ]))
+    .then(([dependencies, asyncDependencies]) => {
         const mainModuleId = dependencies[0].id;
         self._prependPolyfillDependencies(
           dependencies,
-          opts.dev
+          opts.dev,
         );
 
         return {
           mainModuleId: mainModuleId,
-          dependencies: dependencies
+          dependencies: dependencies,
+          asyncDependencies: asyncDependencies,
         };
       }
-    )
+    );
-  );
 };
 
 HasteDependencyResolver.prototype._prependPolyfillDependencies = function(