Add support for module groups to iOS Random Access Bundle format

Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8

react-packager/src/Bundler/Bundle.js

@@ -17,7 +17,7 @@ const crypto = require('crypto');
 const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
 
 class Bundle extends BundleBase {
-  constructor({sourceMapUrl, dev, minify} = {}) {
+  constructor({sourceMapUrl, dev, minify, ramGroups} = {}) {
     super();
     this._sourceMap = false;
     this._sourceMapUrl = sourceMapUrl;
@@ -26,6 +26,7 @@ class Bundle extends BundleBase {
     this._dev = dev;
     this._minify = minify;
 
+    this._ramGroups = ramGroups;
     this._ramBundle = null; // cached RAM Bundle
   }
 
@@ -111,9 +112,17 @@ class Bundle extends BundleBase {
       // separate modules we need to preload from the ones we don't
       const [startupModules, lazyModules] = partition(modules, shouldPreload);
 
+      const ramGroups = this._ramGroups;
+      let groups;
       this._ramBundle = {
         startupModules,
         lazyModules,
+        get groups() {
+          if (!groups) {
+            groups = createGroups(ramGroups || [], lazyModules);
+          }
+          return groups;
+        }
       };
     }
 
@@ -265,6 +274,10 @@ class Bundle extends BundleBase {
     ].join('\n');
   }
 
+  setRamGroups(ramGroups) {
+    this._ramGroups = ramGroups;
+  }
+
   toJSON() {
     this.assertFinalized('Cannot serialize bundle unless finalized');
 
@@ -318,4 +331,89 @@ function partition(array, predicate) {
   return [included, excluded];
 }
 
+function * filter(iterator, predicate) {
+  for (const value of iterator) {
+    if (predicate(value)) {
+      yield value;
+    }
+  }
+}
+
+function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) {
+  seen.add(moduleTransport.id);
+  for (const [, {path}] of moduleTransport.meta.dependencyPairs) {
+    const dependency = moduleTransportsByPath.get(path);
+    if (dependency && !seen.has(dependency.id)) {
+      yield dependency.id;
+      yield * subtree(dependency, moduleTransportsByPath, seen);
+    }
+  }
+}
+
+class ArrayMap extends Map {
+  get(key) {
+    let array = super.get(key);
+    if (!array) {
+      array = [];
+      this.set(key, array);
+    }
+    return array;
+  }
+}
+
+function createGroups(ramGroups, lazyModules) {
+  // build two maps that allow to lookup module data
+  // by path or (numeric) module id;
+  const byPath = new Map();
+  const byId = new Map();
+  lazyModules.forEach(m => {
+    byPath.set(m.sourcePath, m);
+    byId.set(m.id, m.sourcePath);
+  });
+
+  // build a map of group root IDs to an array of module IDs in the group
+  const result = new Map(
+    ramGroups
+      .map(modulePath => {
+        const root = byPath.get(modulePath);
+        if (!root) {
+          throw Error(`Group root ${modulePath} is not part of the bundle`);
+        }
+        return [
+          root.id,
+          // `subtree` yields the IDs of all transitive dependencies of a module
+          new Set(subtree(byPath.get(root.sourcePath), byPath)),
+        ];
+      })
+  );
+
+  if (ramGroups.length > 1) {
+    // build a map of all grouped module IDs to an array of group root IDs
+    const all = new ArrayMap();
+    for (const [parent, children] of result) {
+      for (const module of children) {
+        all.get(module).push(parent);
+      }
+    }
+
+    // find all module IDs that are part of more than one group
+    const doubles = filter(all, ([, parents]) => parents.length > 1);
+    for (const [moduleId, parents] of doubles) {
+      // remove them from their groups
+      parents.forEach(p => result.get(p).delete(moduleId));
+
+      // print a warning for each removed module
+      const parentNames = parents.map(byId.get, byId);
+      const lastName = parentNames.pop();
+      console.warn(
+        `Module ${byId.get(moduleId)} belongs to groups ${
+          parentNames.join(', ')}, and ${lastName
+          }. Removing it from all groups.`
+      );
+    }
+  }
+
+  return result;
+}
+
 module.exports = Bundle;

react-packager/src/Bundler/__tests__/Bundle-test.js

@@ -333,6 +333,114 @@ describe('Bundle', () => {
       expect(deserialized.getMainModuleId()).toEqual(id);
     });
   });
+
+  describe('random access bundle groups:', () => {
+    let moduleTransports;
+    beforeEach(() => {
+      moduleTransports = [
+        transport('Product1', ['React', 'Relay']),
+        transport('React', ['ReactFoo', 'ReactBar']),
+        transport('ReactFoo', ['invariant']),
+        transport('invariant', []),
+        transport('ReactBar', ['cx']),
+        transport('cx', []),
+        transport('OtherFramework', ['OtherFrameworkFoo', 'OtherFrameworkBar']),
+        transport('OtherFrameworkFoo', ['invariant']),
+        transport('OtherFrameworkBar', ['crc32']),
+        transport('crc32', ['OtherFrameworkBar']),
+      ];
+    });
+
+    it('can create a single group', () => {
+      bundle = createBundle([fsLocation('React')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('React'), new Set(['ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))],
+      ]));
+    });
+
+    it('can create two groups', () => {
+      bundle = createBundle([fsLocation('ReactFoo'), fsLocation('ReactBar')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('ReactFoo'), new Set([idFor('invariant')])],
+        [idFor('ReactBar'), new Set([idFor('cx')])],
+      ]));
+    });
+
+    it('can handle circular dependencies', () => {
+      bundle = createBundle([fsLocation('OtherFramework')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([[
+        idFor('OtherFramework'),
+        new Set(['OtherFrameworkFoo', 'invariant', 'OtherFrameworkBar', 'crc32'].map(idFor)),
+      ]]));
+    });
+
+    it('omits modules that are contained by more than one group', () => {
+      bundle = createBundle([fsLocation('React'), fsLocation('OtherFramework')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('React'),
+          new Set(['ReactFoo', 'ReactBar', 'cx'].map(idFor))],
+        [idFor('OtherFramework'),
+          new Set(['OtherFrameworkFoo', 'OtherFrameworkBar', 'crc32'].map(idFor))],
+      ]));
+    });
+
+    it('ignores missing dependencies', () => {
+      bundle = createBundle([fsLocation('Product1')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([[
+        idFor('Product1'),
+        new Set(['React', 'ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))
+      ]]));
+    });
+
+    it('throws for group roots that do not exist', () => {
+      bundle = createBundle([fsLocation('DoesNotExist')]);
+      expect(() => {
+        const {groups} = bundle.getUnbundle(); //eslint-disable-line no-unused-vars
+      }).toThrow(new Error(`Group root ${fsLocation('DoesNotExist')} is not part of the bundle`));
+    });
+
+    function idFor(name) {
+      const {map} = idFor;
+      if (!map) {
+        idFor.map = new Map([[name, 0]]);
+        idFor.next = 1;
+        return 0;
+      }
+
+      if (map.has(name)) {
+        return map.get(name);
+      }
+
+      const id = idFor.next++;
+      map.set(name, id);
+      return id;
+    }
+    function createBundle(ramGroups, options = {}) {
+      const b = new Bundle(Object.assign(options, {ramGroups}));
+      moduleTransports.forEach(t => addModule({bundle: b, ...t}));
+      b.finalize();
+      return b;
+    }
+    function fsLocation(name) {
+      return `/fs/${name}.js`;
+    }
+    function module(name) {
+      return {path: fsLocation(name)};
+    }
+    function transport(name, deps) {
+      return createModuleTransport({
+        name,
+        id: idFor(name),
+        sourcePath: fsLocation(name),
+        meta: {dependencyPairs: deps.map(d => [d, module(d)])},
+      });
+    }
+  });
 });
 
 
@@ -375,7 +483,7 @@ function resolverFor(code, map) {
   };
 }
 
-function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) {
+function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) {
   return bundle.addModule(
     resolverFor(code, map),
     null,
@@ -384,7 +492,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
       code,
       sourceCode,
       sourcePath,
+      id,
       map,
+      meta,
       virtual,
       polyfill,
     }),
@@ -394,9 +504,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
 function createModuleTransport(data) {
   return new ModuleTransport({
     code: '',
-    id: '',
     sourceCode: '',
     sourcePath: '',
+    id: 'id' in data ? data.id : '',
     ...data,
   });
 }

react-packager/src/Bundler/index.js

@@ -393,6 +393,8 @@ class Bundler {
     }
 
     return Promise.resolve(resolutionResponse).then(response => {
+      bundle.setRamGroups(response.transformOptions.transform.ramGroups);
+
       Activity.endEvent(findEventId);
       onResolutionResponse(response);