Add support for module groups to iOS Random Access Bundle format

Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) {
   } = options;
 
   log('start');
-  const {startupModules, lazyModules} = bundle.getUnbundle();
+  const {startupModules, lazyModules, groups} = bundle.getUnbundle();
   log('finish');
 
+  const moduleGroups = ModuleGroups(groups, lazyModules);
   const startupCode = joinModules(startupModules);
 
   log('Writing unbundle output to:', bundleOutput);
   const writeUnbundle = writeBuffers(
     fs.createWriteStream(bundleOutput),
-    buildTableAndContents(startupCode, lazyModules, encoding)
+    buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding)
   ).then(() => log('Done writing unbundle output'));
 
   const sourceMap =
-    buildSourceMapWithMetaData({startupModules, lazyModules});
+    buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups});
 
   return Promise.all([
     writeUnbundle,
@@ -84,7 +85,7 @@ function entryOffset(n) {
   return (2 + n * 2) * SIZEOF_UINT32;
 }
 
-function buildModuleTable(startupCode, buffers) {
+function buildModuleTable(startupCode, buffers, moduleGroups) {
   // table format:
   // - num_entries:      uint_32  number of entries
   // - startup_code_len: uint_32  length of the startup section
@@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) {
   //  - module_offset:   uint_32  offset into the modules blob
   //  - module_length:   uint_32  length of the module code in bytes
 
-  const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0);
+  const moduleIds = Array.from(moduleGroups.modulesById.keys());
+  const maxId = moduleIds.reduce((max, id) => Math.max(max, id));
   const numEntries = maxId + 1;
   const table = new Buffer(entryOffset(numEntries)).fill(0);
 
@@ -107,23 +109,50 @@ function buildModuleTable(startupCode, buffers) {
   // entries
   let codeOffset = startupCode.length;
   buffers.forEach(({id, buffer}) => {
-    const offset = entryOffset(id);
-    // module_offset
-    table.writeUInt32LE(codeOffset, offset);
-    // module_length
-    table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+    const idsInGroup = moduleGroups.groups.has(id)
+      ? [id].concat(Array.from(moduleGroups.groups.get(id)))
+      : [id];
+
+    idsInGroup.forEach(moduleId => {
+      const offset = entryOffset(moduleId);
+      // module_offset
+      table.writeUInt32LE(codeOffset, offset);
+      // module_length
+      table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+    });
     codeOffset += buffer.length;
   });
 
   return table;
 }
 
-function buildModuleBuffers(modules, encoding) {
-  return modules.map(
-    module => moduleToBuffer(module.id, module.code, encoding));
+function groupCode(rootCode, moduleGroup, modulesById) {
+  if (!moduleGroup || !moduleGroup.size) {
+    return rootCode;
+  }
+  const code = [rootCode];
+  for (const id of moduleGroup) {
+    code.push(modulesById.get(id).code);
+  }
+
+  return code.join('\n');
 }
 
-function buildTableAndContents(startupCode, modules, encoding) {
+function buildModuleBuffers(modules, moduleGroups, encoding) {
+  return modules
+    .filter(m => !moduleGroups.modulesInGroups.has(m.id))
+    .map(({id, code}) => moduleToBuffer(
+      id,
+      groupCode(
+        code,
+        moduleGroups.groups.get(id),
+        moduleGroups.modulesById,
+      ),
+      encoding
+    ));
+}
+
+function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
   // file contents layout:
   // - magic number      char[4]  0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
   // - offset table      table    see `buildModuleTables`
@@ -131,8 +160,8 @@ function buildTableAndContents(startupCode, modules, encoding) {
   //                              the startup code
 
   const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding);
-  const moduleBuffers = buildModuleBuffers(modules, encoding);
-  const table = buildModuleTable(startupCodeBuffer, moduleBuffers);
+  const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding);
+  const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups);
 
   return [
     fileHeader,
@@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) {
   ].concat(moduleBuffers.map(({buffer}) => buffer));
 }
 
+function ModuleGroups(groups, modules) {
+  return {
+    groups,
+    modulesById: new Map(modules.map(m => [m.id, m])),
+    modulesInGroups: new Set(concat(groups.values())),
+  };
+}
+
+function * concat(iterators) {
+  for (const it of iterators) {
+    yield * it;
+  }
+}
+
 module.exports = saveAsIndexedFile;

local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js

@@ -10,13 +10,14 @@
 
 const {combineSourceMaps, joinModules} = require('./util');
 
-module.exports = ({startupModules, lazyModules}) => {
+module.exports = ({startupModules, lazyModules, moduleGroups}) => {
   const startupModule = {
     code: joinModules(startupModules),
     map: combineSourceMaps({modules: startupModules}),
   };
   return combineSourceMaps({
     modules: [startupModule].concat(lazyModules),
+    moduleGroups,
     withCustomOffsets: true,
   });
 };

local-cli/bundle/output/unbundle/util.js

@@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1;
 
 const Section = (line, column, map) => ({map, offset: {line, column}});
 
-function combineSourceMaps({modules, withCustomOffsets}) {
+function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) {
   let offsets;
   const sections = [];
   const sourceMap = {
@@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) {
 
   let line = 0;
   modules.forEach(({code, id, map, name}) => {
-    const hasOffset = withCustomOffsets && id != null;
-    const column = hasOffset ? wrapperEnd(code) : 0;
+    let column = 0;
+    let hasOffset = false;
+    let group;
+    let groupLines = 0;
+
+    if (withCustomOffsets) {
+      if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
+        // this is a module appended to another module
+        return;
+      }
+
+      if (moduleGroups && moduleGroups.groups.has(id)) {
+        group = moduleGroups.groups.get(id);
+        const otherModules = Array.from(group).map(
+          moduleId => moduleGroups.modulesById.get(moduleId));
+        otherModules.forEach(m => {
+          groupLines += countLines(m.code);
+        });
+        map = combineSourceMaps({
+          modules: [{code, id, map, name}].concat(otherModules),
+        });
+      }
+
+      hasOffset = id != null;
+      column = wrapperEnd(code);
+    }
+
     sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
     if (hasOffset) {
       offsets[id] = line;
+      for (const moduleId of group || []) {
+        offsets[moduleId] = line;
+      }
     }
-    line += countLines(code);
+    line += countLines(code) + groupLines;
   });
 
   return sourceMap;

packager/react-packager/src/Bundler/Bundle.js

@@ -17,7 +17,7 @@ const crypto = require('crypto');
 const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
 
 class Bundle extends BundleBase {
-  constructor({sourceMapUrl, dev, minify} = {}) {
+  constructor({sourceMapUrl, dev, minify, ramGroups} = {}) {
     super();
     this._sourceMap = false;
     this._sourceMapUrl = sourceMapUrl;
@@ -26,6 +26,7 @@ class Bundle extends BundleBase {
     this._dev = dev;
     this._minify = minify;
 
+    this._ramGroups = ramGroups;
     this._ramBundle = null; // cached RAM Bundle
   }
 
@@ -111,9 +112,17 @@ class Bundle extends BundleBase {
       // separate modules we need to preload from the ones we don't
       const [startupModules, lazyModules] = partition(modules, shouldPreload);
 
+      const ramGroups = this._ramGroups;
+      let groups;
       this._ramBundle = {
         startupModules,
         lazyModules,
+        get groups() {
+          if (!groups) {
+            groups = createGroups(ramGroups || [], lazyModules);
+          }
+          return groups;
+        }
       };
     }
 
@@ -265,6 +274,10 @@ class Bundle extends BundleBase {
     ].join('\n');
   }
 
+  setRamGroups(ramGroups) {
+    this._ramGroups = ramGroups;
+  }
+
   toJSON() {
     this.assertFinalized('Cannot serialize bundle unless finalized');
 
@@ -318,4 +331,89 @@ function partition(array, predicate) {
   return [included, excluded];
 }
 
+function * filter(iterator, predicate) {
+  for (const value of iterator) {
+    if (predicate(value)) {
+      yield value;
+    }
+  }
+}
+
+function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) {
+  seen.add(moduleTransport.id);
+  for (const [, {path}] of moduleTransport.meta.dependencyPairs) {
+    const dependency = moduleTransportsByPath.get(path);
+    if (dependency && !seen.has(dependency.id)) {
+      yield dependency.id;
+      yield * subtree(dependency, moduleTransportsByPath, seen);
+    }
+  }
+}
+
+class ArrayMap extends Map {
+  get(key) {
+    let array = super.get(key);
+    if (!array) {
+      array = [];
+      this.set(key, array);
+    }
+    return array;
+  }
+}
+
+function createGroups(ramGroups, lazyModules) {
+  // build two maps that allow to lookup module data
+  // by path or (numeric) module id;
+  const byPath = new Map();
+  const byId = new Map();
+  lazyModules.forEach(m => {
+    byPath.set(m.sourcePath, m);
+    byId.set(m.id, m.sourcePath);
+  });
+
+  // build a map of group root IDs to an array of module IDs in the group
+  const result = new Map(
+    ramGroups
+      .map(modulePath => {
+        const root = byPath.get(modulePath);
+        if (!root) {
+          throw Error(`Group root ${modulePath} is not part of the bundle`);
+        }
+        return [
+          root.id,
+          // `subtree` yields the IDs of all transitive dependencies of a module
+          new Set(subtree(byPath.get(root.sourcePath), byPath)),
+        ];
+      })
+  );
+
+  if (ramGroups.length > 1) {
+    // build a map of all grouped module IDs to an array of group root IDs
+    const all = new ArrayMap();
+    for (const [parent, children] of result) {
+      for (const module of children) {
+        all.get(module).push(parent);
+      }
+    }
+
+    // find all module IDs that are part of more than one group
+    const doubles = filter(all, ([, parents]) => parents.length > 1);
+    for (const [moduleId, parents] of doubles) {
+      // remove them from their groups
+      parents.forEach(p => result.get(p).delete(moduleId));
+
+      // print a warning for each removed module
+      const parentNames = parents.map(byId.get, byId);
+      const lastName = parentNames.pop();
+      console.warn(
+        `Module ${byId.get(moduleId)} belongs to groups ${
+          parentNames.join(', ')}, and ${lastName
+          }. Removing it from all groups.`
+      );
+    }
+  }
+
+  return result;
+}
+
 module.exports = Bundle;

packager/react-packager/src/Bundler/__tests__/Bundle-test.js

@@ -333,6 +333,114 @@ describe('Bundle', () => {
       expect(deserialized.getMainModuleId()).toEqual(id);
     });
   });
+
+  describe('random access bundle groups:', () => {
+    let moduleTransports;
+    beforeEach(() => {
+      moduleTransports = [
+        transport('Product1', ['React', 'Relay']),
+        transport('React', ['ReactFoo', 'ReactBar']),
+        transport('ReactFoo', ['invariant']),
+        transport('invariant', []),
+        transport('ReactBar', ['cx']),
+        transport('cx', []),
+        transport('OtherFramework', ['OtherFrameworkFoo', 'OtherFrameworkBar']),
+        transport('OtherFrameworkFoo', ['invariant']),
+        transport('OtherFrameworkBar', ['crc32']),
+        transport('crc32', ['OtherFrameworkBar']),
+      ];
+    });
+
+    it('can create a single group', () => {
+      bundle = createBundle([fsLocation('React')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('React'), new Set(['ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))],
+      ]));
+    });
+
+    it('can create two groups', () => {
+      bundle = createBundle([fsLocation('ReactFoo'), fsLocation('ReactBar')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('ReactFoo'), new Set([idFor('invariant')])],
+        [idFor('ReactBar'), new Set([idFor('cx')])],
+      ]));
+    });
+
+    it('can handle circular dependencies', () => {
+      bundle = createBundle([fsLocation('OtherFramework')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([[
+        idFor('OtherFramework'),
+        new Set(['OtherFrameworkFoo', 'invariant', 'OtherFrameworkBar', 'crc32'].map(idFor)),
+      ]]));
+    });
+
+    it('omits modules that are contained by more than one group', () => {
+      bundle = createBundle([fsLocation('React'), fsLocation('OtherFramework')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([
+        [idFor('React'),
+          new Set(['ReactFoo', 'ReactBar', 'cx'].map(idFor))],
+        [idFor('OtherFramework'),
+          new Set(['OtherFrameworkFoo', 'OtherFrameworkBar', 'crc32'].map(idFor))],
+      ]));
+    });
+
+    it('ignores missing dependencies', () => {
+      bundle = createBundle([fsLocation('Product1')]);
+      const {groups} = bundle.getUnbundle();
+      expect(groups).toEqual(new Map([[
+        idFor('Product1'),
+        new Set(['React', 'ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))
+      ]]));
+    });
+
+    it('throws for group roots that do not exist', () => {
+      bundle = createBundle([fsLocation('DoesNotExist')]);
+      expect(() => {
+        const {groups} = bundle.getUnbundle(); //eslint-disable-line no-unused-vars
+      }).toThrow(new Error(`Group root ${fsLocation('DoesNotExist')} is not part of the bundle`));
+    });
+
+    function idFor(name) {
+      const {map} = idFor;
+      if (!map) {
+        idFor.map = new Map([[name, 0]]);
+        idFor.next = 1;
+        return 0;
+      }
+
+      if (map.has(name)) {
+        return map.get(name);
+      }
+
+      const id = idFor.next++;
+      map.set(name, id);
+      return id;
+    }
+    function createBundle(ramGroups, options = {}) {
+      const b = new Bundle(Object.assign(options, {ramGroups}));
+      moduleTransports.forEach(t => addModule({bundle: b, ...t}));
+      b.finalize();
+      return b;
+    }
+    function fsLocation(name) {
+      return `/fs/${name}.js`;
+    }
+    function module(name) {
+      return {path: fsLocation(name)};
+    }
+    function transport(name, deps) {
+      return createModuleTransport({
+        name,
+        id: idFor(name),
+        sourcePath: fsLocation(name),
+        meta: {dependencyPairs: deps.map(d => [d, module(d)])},
+      });
+    }
+  });
 });
 
 
@@ -375,7 +483,7 @@ function resolverFor(code, map) {
   };
 }
 
-function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) {
+function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) {
   return bundle.addModule(
     resolverFor(code, map),
     null,
@@ -384,7 +492,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
       code,
       sourceCode,
       sourcePath,
+      id,
       map,
+      meta,
       virtual,
       polyfill,
     }),
@@ -394,9 +504,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
 function createModuleTransport(data) {
   return new ModuleTransport({
     code: '',
-    id: '',
     sourceCode: '',
     sourcePath: '',
+    id: 'id' in data ? data.id : '',
     ...data,
   });
 }

packager/react-packager/src/Bundler/index.js

@@ -393,6 +393,8 @@ class Bundler {
     }
 
     return Promise.resolve(resolutionResponse).then(response => {
+      bundle.setRamGroups(response.transformOptions.transform.ramGroups);
+
       Activity.endEvent(findEventId);
       onResolutionResponse(response);