From fb02e5d21629785602facb2f1c4703f41bb1c008 Mon Sep 17 00:00:00 2001 From: David Aurelio Date: Fri, 16 Sep 2016 10:03:34 -0700 Subject: [PATCH] Add support for module groups to iOS Random Access Bundle format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code. A require call for any grouped module will load all modules in that group. Files contained by multiple groups are deoptimized (i.e. bundled as individual script) Reviewed By: martinbigio Differential Revision: D3841780 fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8 --- react-packager/src/Bundler/Bundle.js | 100 ++++++++++++++- .../src/Bundler/__tests__/Bundle-test.js | 114 +++++++++++++++++- react-packager/src/Bundler/index.js | 2 + 3 files changed, 213 insertions(+), 3 deletions(-) diff --git a/react-packager/src/Bundler/Bundle.js b/react-packager/src/Bundler/Bundle.js index acca6649..7bed098d 100644 --- a/react-packager/src/Bundler/Bundle.js +++ b/react-packager/src/Bundler/Bundle.js @@ -17,7 +17,7 @@ const crypto = require('crypto'); const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL='; class Bundle extends BundleBase { - constructor({sourceMapUrl, dev, minify} = {}) { + constructor({sourceMapUrl, dev, minify, ramGroups} = {}) { super(); this._sourceMap = false; this._sourceMapUrl = sourceMapUrl; @@ -26,6 +26,7 @@ class Bundle extends BundleBase { this._dev = dev; this._minify = minify; + this._ramGroups = ramGroups; this._ramBundle = null; // cached RAM Bundle } @@ -111,9 +112,17 @@ class Bundle extends BundleBase { // separate modules we need to preload from the ones we don't const [startupModules, lazyModules] = partition(modules, shouldPreload); + const ramGroups = this._ramGroups; + let groups; this._ramBundle = { startupModules, lazyModules, + get groups() { + if (!groups) { + groups = createGroups(ramGroups || [], lazyModules); + } + return groups; + } }; } @@ -265,6 +274,10 @@ class Bundle extends BundleBase { ].join('\n'); } + setRamGroups(ramGroups) { + this._ramGroups = ramGroups; + } + toJSON() { this.assertFinalized('Cannot serialize bundle unless finalized'); @@ -318,4 +331,89 @@ function partition(array, predicate) { return [included, excluded]; } +function * filter(iterator, predicate) { + for (const value of iterator) { + if (predicate(value)) { + yield value; + } + } +} + +function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) { + seen.add(moduleTransport.id); + for (const [, {path}] of moduleTransport.meta.dependencyPairs) { + const dependency = moduleTransportsByPath.get(path); + if (dependency && !seen.has(dependency.id)) { + yield dependency.id; + yield * subtree(dependency, moduleTransportsByPath, seen); + } + } +} + +class ArrayMap extends Map { + get(key) { + let array = super.get(key); + if (!array) { + array = []; + this.set(key, array); + } + return array; + } +} + +function createGroups(ramGroups, lazyModules) { + // build two maps that allow to lookup module data + // by path or (numeric) module id; + const byPath = new Map(); + const byId = new Map(); + lazyModules.forEach(m => { + byPath.set(m.sourcePath, m); + byId.set(m.id, m.sourcePath); + }); + + // build a map of group root IDs to an array of module IDs in the group + const result = new Map( + ramGroups + .map(modulePath => { + const root = byPath.get(modulePath); + if (!root) { + throw Error(`Group root ${modulePath} is not part of the bundle`); + } + return [ + root.id, + // `subtree` yields the IDs of all transitive dependencies of a module + new Set(subtree(byPath.get(root.sourcePath), byPath)), + ]; + }) + ); + + if (ramGroups.length > 1) { + // build a map of all grouped module IDs to an array of group root IDs + const all = new ArrayMap(); + for (const [parent, children] of result) { + for (const module of children) { + all.get(module).push(parent); + } + } + + // find all module IDs that are part of more than one group + const doubles = filter(all, ([, parents]) => parents.length > 1); + for (const [moduleId, parents] of doubles) { + // remove them from their groups + parents.forEach(p => result.get(p).delete(moduleId)); + + // print a warning for each removed module + const parentNames = parents.map(byId.get, byId); + const lastName = parentNames.pop(); + console.warn( + `Module ${byId.get(moduleId)} belongs to groups ${ + parentNames.join(', ')}, and ${lastName + }. Removing it from all groups.` + ); + } + } + + return result; +} + module.exports = Bundle; diff --git a/react-packager/src/Bundler/__tests__/Bundle-test.js b/react-packager/src/Bundler/__tests__/Bundle-test.js index 1f6f68ff..a043c164 100644 --- a/react-packager/src/Bundler/__tests__/Bundle-test.js +++ b/react-packager/src/Bundler/__tests__/Bundle-test.js @@ -333,6 +333,114 @@ describe('Bundle', () => { expect(deserialized.getMainModuleId()).toEqual(id); }); }); + + describe('random access bundle groups:', () => { + let moduleTransports; + beforeEach(() => { + moduleTransports = [ + transport('Product1', ['React', 'Relay']), + transport('React', ['ReactFoo', 'ReactBar']), + transport('ReactFoo', ['invariant']), + transport('invariant', []), + transport('ReactBar', ['cx']), + transport('cx', []), + transport('OtherFramework', ['OtherFrameworkFoo', 'OtherFrameworkBar']), + transport('OtherFrameworkFoo', ['invariant']), + transport('OtherFrameworkBar', ['crc32']), + transport('crc32', ['OtherFrameworkBar']), + ]; + }); + + it('can create a single group', () => { + bundle = createBundle([fsLocation('React')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('React'), new Set(['ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))], + ])); + }); + + it('can create two groups', () => { + bundle = createBundle([fsLocation('ReactFoo'), fsLocation('ReactBar')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('ReactFoo'), new Set([idFor('invariant')])], + [idFor('ReactBar'), new Set([idFor('cx')])], + ])); + }); + + it('can handle circular dependencies', () => { + bundle = createBundle([fsLocation('OtherFramework')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([[ + idFor('OtherFramework'), + new Set(['OtherFrameworkFoo', 'invariant', 'OtherFrameworkBar', 'crc32'].map(idFor)), + ]])); + }); + + it('omits modules that are contained by more than one group', () => { + bundle = createBundle([fsLocation('React'), fsLocation('OtherFramework')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('React'), + new Set(['ReactFoo', 'ReactBar', 'cx'].map(idFor))], + [idFor('OtherFramework'), + new Set(['OtherFrameworkFoo', 'OtherFrameworkBar', 'crc32'].map(idFor))], + ])); + }); + + it('ignores missing dependencies', () => { + bundle = createBundle([fsLocation('Product1')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([[ + idFor('Product1'), + new Set(['React', 'ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor)) + ]])); + }); + + it('throws for group roots that do not exist', () => { + bundle = createBundle([fsLocation('DoesNotExist')]); + expect(() => { + const {groups} = bundle.getUnbundle(); //eslint-disable-line no-unused-vars + }).toThrow(new Error(`Group root ${fsLocation('DoesNotExist')} is not part of the bundle`)); + }); + + function idFor(name) { + const {map} = idFor; + if (!map) { + idFor.map = new Map([[name, 0]]); + idFor.next = 1; + return 0; + } + + if (map.has(name)) { + return map.get(name); + } + + const id = idFor.next++; + map.set(name, id); + return id; + } + function createBundle(ramGroups, options = {}) { + const b = new Bundle(Object.assign(options, {ramGroups})); + moduleTransports.forEach(t => addModule({bundle: b, ...t})); + b.finalize(); + return b; + } + function fsLocation(name) { + return `/fs/${name}.js`; + } + function module(name) { + return {path: fsLocation(name)}; + } + function transport(name, deps) { + return createModuleTransport({ + name, + id: idFor(name), + sourcePath: fsLocation(name), + meta: {dependencyPairs: deps.map(d => [d, module(d)])}, + }); + } + }); }); @@ -375,7 +483,7 @@ function resolverFor(code, map) { }; } -function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) { +function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) { return bundle.addModule( resolverFor(code, map), null, @@ -384,7 +492,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill code, sourceCode, sourcePath, + id, map, + meta, virtual, polyfill, }), @@ -394,9 +504,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill function createModuleTransport(data) { return new ModuleTransport({ code: '', - id: '', sourceCode: '', sourcePath: '', + id: 'id' in data ? data.id : '', ...data, }); } diff --git a/react-packager/src/Bundler/index.js b/react-packager/src/Bundler/index.js index 2d640dae..dfa7680f 100644 --- a/react-packager/src/Bundler/index.js +++ b/react-packager/src/Bundler/index.js @@ -393,6 +393,8 @@ class Bundler { } return Promise.resolve(resolutionResponse).then(response => { + bundle.setRamGroups(response.transformOptions.transform.ramGroups); + Activity.endEvent(findEventId); onResolutionResponse(response);