[react-packager] Cache BundlesLayout

react-packager/src/Bundler/index.js

@@ -13,6 +13,7 @@ const fs = require('fs');
 const path = require('path');
 const Promise = require('promise');
 const ProgressBar = require('progress');
+const BundlesLayout = require('../BundlesLayout');
 const Cache = require('../Cache');
 const Transformer = require('../JSTransformer');
 const DependencyResolver = require('../DependencyResolver');
@@ -104,6 +105,13 @@ class Bundler {
       cache: this._cache,
     });
 
+    this._bundlesLayout = new BundlesLayout({
+      dependencyResolver: this._resolver,
+      resetCache: opts.resetCache,
+      cacheVersion: opts.cacheVersion,
+      projectRoots: opts.projectRoots,
+    });
+
     this._transformer = new Transformer({
       projectRoots: opts.projectRoots,
       blacklistRE: opts.blacklistRE,
@@ -120,6 +128,10 @@ class Bundler {
     return this._cache.end();
   }
 
+  getLayout(main, isDev) {
+    return this._bundlesLayout.generateLayout(main, isDev);
+  }
+
   bundle(main, runModule, sourceMapUrl, isDev, platform) {
     const bundle = new Bundle(sourceMapUrl);
     const findEventId = Activity.startEvent('find dependencies');

react-packager/src/BundlesLayout/__tests__/BundlesLayout-test.js

@@ -8,31 +8,35 @@
  */
 'use strict';
 
-jest
+jest.dontMock('../index');
-  .dontMock('../index');
+jest.mock('fs');
 
 const Promise = require('promise');
 
 describe('BundlesLayout', () => {
-  var BundlesLayout;
+  let BundlesLayout;
-  var DependencyResolver;
+  let DependencyResolver;
+  let loadCacheSync;
 
   beforeEach(() => {
     BundlesLayout = require('../index');
     DependencyResolver = require('../../DependencyResolver');
+    loadCacheSync = require('../../lib/loadCacheSync');
   });
 
-  describe('generate', () => {
+  function newBundlesLayout(options) {
-    function newBundlesLayout() {
+    return new BundlesLayout(Object.assign({
-      return new BundlesLayout({
+      projectRoots: ['/root'],
       dependencyResolver: new DependencyResolver(),
-      });
+    }, options));
   }
 
+  describe('layout', () => {
     function isPolyfill() {
       return false;
     }
 
+    describe('getLayout', () => {
       function dep(path) {
         return {
           path: path,
@@ -58,7 +62,9 @@ describe('BundlesLayout', () => {
           }
         });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        return newBundlesLayout({resetCache: true})
+          .getLayout('/root/index.js')
+          .then(bundles =>
             expect(bundles).toEqual({
               id: 'bundle.0',
               modules: ['/root/index.js', '/root/a.js'],
@@ -85,7 +91,9 @@ describe('BundlesLayout', () => {
           }
         });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        return newBundlesLayout({resetCache: true})
+          .getLayout('/root/index.js')
+          .then(bundles =>
             expect(bundles).toEqual({
               id: 'bundle.0',
               modules: ['/root/index.js'],
@@ -121,7 +129,9 @@ describe('BundlesLayout', () => {
           }
         });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        return newBundlesLayout({resetCache: true})
+          .getLayout('/root/index.js')
+          .then(bundles =>
             expect(bundles).toEqual({
               id: 'bundle.0',
               modules: ['/root/index.js'],
@@ -161,7 +171,9 @@ describe('BundlesLayout', () => {
           }
         });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+        return newBundlesLayout({resetCache: true})
+          .getLayout('/root/index.js')
+          .then(bundles =>
             expect(bundles).toEqual({
               id: 'bundle.0',
               modules: ['/root/index.js'],
@@ -197,8 +209,10 @@ describe('BundlesLayout', () => {
           }
         });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(
+        return newBundlesLayout({resetCache: true})
-        bundles => expect(bundles).toEqual({
+          .getLayout('/root/index.js')
+          .then(bundles =>
+            expect(bundles).toEqual({
               id: 'bundle.0',
               modules: ['/root/index.js', '/root/a.js'],
               children: [{
@@ -243,8 +257,8 @@ describe('BundlesLayout', () => {
           }
         });
 
-      var layout = newBundlesLayout();
+        var layout = newBundlesLayout({resetCache: true});
-      return layout.generateLayout(['/root/index.js']).then(() => {
+        return layout.getLayout('/root/index.js').then(() => {
           expect(layout.getBundleIDForModule('/root/index.js')).toBe('bundle.0');
           expect(layout.getBundleIDForModule('/root/a.js')).toBe('bundle.0');
           expect(layout.getBundleIDForModule('/root/b.js')).toBe('bundle.0.1');
@@ -253,4 +267,61 @@ describe('BundlesLayout', () => {
         });
       });
     });
+  });
+
+  describe('cache', () => {
+    beforeEach(() => {
+      loadCacheSync.mockReturnValue({
+        '/root/index.js': {
+          id: 'bundle.0',
+          modules: ['/root/index.js'],
+          children: [{
+            id: 'bundle.0.1',
+            modules: ['/root/a.js'],
+            children: [],
+          }],
+        },
+        '/root/b.js': {
+          id: 'bundle.2',
+          modules: ['/root/b.js'],
+          children: [],
+        },
+      });
+    });
+
+    pit('should load layouts', () => {
+      const layout = newBundlesLayout({ resetCache: false });
+
+      return Promise
+        .all([
+          layout.getLayout('/root/index.js'),
+          layout.getLayout('/root/b.js'),
+        ])
+        .then(([layoutIndex, layoutB]) => {
+          expect(layoutIndex).toEqual({
+            id: 'bundle.0',
+            modules: ['/root/index.js'],
+            children: [{
+              id: 'bundle.0.1',
+              modules: ['/root/a.js'],
+              children: [],
+            }],
+          });
+
+          expect(layoutB).toEqual({
+            id: 'bundle.2',
+            modules: ['/root/b.js'],
+            children: [],
+          });
+        });
+    });
+
+    it('should load moduleToBundle map', () => {
+      const layout = newBundlesLayout({ resetCache: false });
+
+      expect(layout.getBundleIDForModule('/root/index.js')).toBe('bundle.0');
+      expect(layout.getBundleIDForModule('/root/a.js')).toBe('bundle.0.1');
+      expect(layout.getBundleIDForModule('/root/b.js')).toBe('bundle.2');
+    });
+  });
 });

react-packager/src/BundlesLayout/__tests__/BundlesLayoutIntegration-test.js

@@ -75,7 +75,11 @@ describe('BundlesLayout', () => {
         assetRoots: ['/root'],
       });
 
-      return new BundlesLayout({dependencyResolver: resolver});
+      return new BundlesLayout({
+        dependencyResolver: resolver,
+        resetCache: true,
+        projectRoots: ['/root', '/' + __dirname.split('/')[1]],
+      });
     }
 
     function stripPolyfills(bundle) {
@@ -114,7 +118,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -140,7 +144,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -166,7 +170,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -201,7 +205,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -242,7 +246,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -282,7 +286,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -323,7 +327,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -370,7 +374,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -408,7 +412,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -446,7 +450,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -480,7 +484,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -512,7 +516,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -539,7 +543,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',
@@ -576,7 +580,7 @@ describe('BundlesLayout', () => {
         }
       });
 
-      return newBundlesLayout().generateLayout(['/root/index.js']).then(bundles =>
+      return newBundlesLayout().getLayout('/root/index.js').then(bundles =>
         stripPolyfills(bundles).then(resolvedBundles =>
           expect(resolvedBundles).toEqual({
             id: 'bundle.0',

react-packager/src/BundlesLayout/index.js

@@ -8,14 +8,33 @@
  */
 'use strict';
 
+const Activity = require('../Activity');
+
 const _ = require('underscore');
 const declareOpts = require('../lib/declareOpts');
+const fs = require('fs');
+const getCacheFilePath = require('../lib/getCacheFilePath');
+const loadCacheSync = require('../lib/loadCacheSync');
+const version = require('../../../../package.json').version;
+const path = require('path');
 
 const validateOpts = declareOpts({
   dependencyResolver: {
     type: 'object',
     required: true,
   },
+  resetCache: {
+    type: 'boolean',
+    default: false,
+  },
+  cacheVersion: {
+    type: 'string',
+    default: '1.0',
+  },
+  projectRoots: {
+    type: 'array',
+    required: true,
+  },
 });
 
 const BUNDLE_PREFIX = 'bundle';
@@ -29,19 +48,37 @@ class BundlesLayout {
     const opts = validateOpts(options);
     this._resolver = opts.dependencyResolver;
 
+    // Cache in which bundle is each module.
     this._moduleToBundle = Object.create(null);
+
+    // Cache the bundles layouts for each entry point. This entries
+    // are not evicted unless the user explicitly specifies so as
+    // computing them is pretty expensive
+    this._layouts = Object.create(null);
+
+    // TODO: watch for file creations and removals to update this caches
+
+    this._cacheFilePath = this._getCacheFilePath(opts);
+    if (!opts.resetCache) {
+      this._loadCacheSync(this._cacheFilePath);
+    } else {
+      this._persistCacheEventually();
+    }
   }
 
-  generateLayout(entryPaths, isDev) {
+  getLayout(entryPath, isDev) {
+    if (this._layouts[entryPath]) {
+      return this._layouts[entryPath];
+    }
     var currentBundleID =  0;
     const rootBundle = {
       id: BUNDLE_PREFIX + '.' + currentBundleID++,
       modules: [],
       children: [],
     };
-    var pending = [{paths: entryPaths, bundle: rootBundle}];
+    var pending = [{paths: [entryPath], bundle: rootBundle}];
 
-    return promiseWhile(
+    this._layouts[entryPath] = promiseWhile(
       () => pending.length > 0,
       () => rootBundle,
       () => {
@@ -62,6 +99,9 @@ class BundlesLayout {
             if (dependencies.length > 0) {
               bundle.modules = dependencies;
             }
+
+            // persist changes to layouts
+            this._persistCacheEventually();
           },
           index => {
             const pendingSyncDep = pendingSyncDeps.shift();
@@ -90,11 +130,71 @@ class BundlesLayout {
         );
       },
     );
+
+    return this._layouts[entryPath];
   }
 
   getBundleIDForModule(path) {
     return this._moduleToBundle[path];
   }
+
+  _loadCacheSync(cachePath) {
+    const loadCacheId = Activity.startEvent('Loading bundles layout');
+    const cacheOnDisk = loadCacheSync(cachePath);
+
+    // TODO: create single-module bundles for unexistent modules
+    // TODO: remove modules that no longer exist
+    Object.keys(cacheOnDisk).forEach(entryPath => {
+      this._layouts[entryPath] = Promise.resolve(cacheOnDisk[entryPath]);
+      this._fillModuleToBundleMap(cacheOnDisk[entryPath]);
+    });
+
+    Activity.endEvent(loadCacheId);
+  }
+
+  _fillModuleToBundleMap(bundle) {
+    bundle.modules.forEach(module => this._moduleToBundle[module] = bundle.id);
+    bundle.children.forEach(child => this._fillModuleToBundleMap(child));
+  }
+
+  _persistCacheEventually() {
+    _.debounce(
+      this._persistCache.bind(this),
+      2000,
+    );
+  }
+
+  _persistCache() {
+    if (this._persisting !== null) {
+      return this._persisting;
+    }
+
+    this._persisting = Promise
+      .all(_.values(this._layouts))
+      .then(bundlesLayout => {
+        var json = Object.create(null);
+        Object.keys(this._layouts).forEach((p, i) =>
+          json[p] = bundlesLayout[i]
+        );
+
+        return Promise.denodeify(fs.writeFile)(
+          this._cacheFilepath,
+          JSON.stringify(json),
+        );
+      })
+      .then(() => this._persisting = null);
+
+    return this._persisting;
+  }
+
+  _getCacheFilePath(options) {
+    return getCacheFilePath(
+      'react-packager-bundles-cache-',
+      version,
+      options.projectRoots.join(',').split(path.sep).join('-'),
+      options.cacheVersion || '0',
+    );
+  }
 }
 
 // Runs the body Promise meanwhile the condition callback is satisfied.

react-packager/src/Cache/__tests__/Cache-test.js

@@ -11,7 +11,9 @@
 jest
   .dontMock('underscore')
   .dontMock('absolute-path')
-  .dontMock('../');
+  .dontMock('../')
+  .dontMock('../../lib/loadCacheSync')
+  .dontMock('../../lib/getCacheFilePath');
 
 jest
   .mock('os')

react-packager/src/Cache/index.js

@@ -8,17 +8,17 @@
  */
 'use strict';
 
-var _ = require('underscore');
+const Promise = require('promise');
-var crypto = require('crypto');
+const _ = require('underscore');
-var declareOpts = require('../lib/declareOpts');
+const declareOpts = require('../lib/declareOpts');
-var fs = require('fs');
+const fs = require('fs');
-var isAbsolutePath = require('absolute-path');
+const getCacheFilePath = require('../lib/getCacheFilePath');
-var path = require('path');
+const isAbsolutePath = require('absolute-path');
-var Promise = require('promise');
+const loadCacheSync = require('../lib/loadCacheSync');
-var tmpdir = require('os').tmpDir();
+const path = require('path');
-var version = require('../../../../package.json').version;
+const version = require('../../../../package.json').version;
 
-var validateOpts = declareOpts({
+const validateOpts = declareOpts({
   resetCache: {
     type: 'boolean',
     default: false,
@@ -164,21 +164,7 @@ class Cache {
 
   _loadCacheSync(cachePath) {
     var ret = Object.create(null);
-    if (!fs.existsSync(cachePath)) {
+    var cacheOnDisk = loadCacheSync(cachePath);
-      return ret;
-    }
-
-    var cacheOnDisk;
-    try {
-      cacheOnDisk = JSON.parse(fs.readFileSync(cachePath));
-    } catch (e) {
-      if (e instanceof SyntaxError) {
-        console.warn('Unable to parse cache file. Will clear and continue.');
-        fs.unlinkSync(cachePath);
-        return ret;
-      }
-      throw e;
-    }
 
     // Filter outdated cache and convert to promises.
     Object.keys(cacheOnDisk).forEach(key => {
@@ -203,20 +189,13 @@ class Cache {
   }
 
   _getCacheFilePath(options) {
-    var hash = crypto.createHash('md5');
+    return getCacheFilePath(
-    hash.update(version);
+      'react-packager-cache-',
-
+      version,
-    var roots = options.projectRoots.join(',').split(path.sep).join('-');
+      options.projectRoots.join(',').split(path.sep).join('-'),
-    hash.update(roots);
+      options.cacheVersion || '0',
-
+      options.transformModulePath,
-    var cacheVersion = options.cacheVersion || '0';
+    );
-    hash.update(cacheVersion);
-
-    hash.update(options.transformModulePath);
-
-    var name = 'react-packager-cache-' + hash.digest('hex');
-
-    return path.join(tmpdir, name);
   }
 }
 

react-packager/src/lib/getCacheFilePath.js

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const crypto = require('crypto');
+const path = require('path');
+const tmpdir = require('os').tmpDir();
+
+function getCacheFilePath(args) {
+  args = Array.prototype.slice.call(args);
+  const prefix = args.shift();
+
+  let hash = crypto.createHash('md5');
+  args.forEach(arg => hash.update(arg));
+
+  return path.join(tmpdir, prefix + hash.digest('hex'));
+}
+
+module.exports = getCacheFilePath;

react-packager/src/lib/loadCacheSync.js

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const fs = require('fs');
+
+function loadCacheSync(cachePath) {
+  if (!fs.existsSync(cachePath)) {
+    return Object.create(null);
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(cachePath));
+  } catch (e) {
+    if (e instanceof SyntaxError) {
+      console.warn('Unable to parse cache file. Will clear and continue.');
+      fs.unlinkSync(cachePath);
+      return Object.create(null);
+    }
+    throw e;
+  }
+}
+
+module.exports = loadCacheSync;