diff --git a/local-cli/bundle/output/unbundle/as-indexed-file.js b/local-cli/bundle/output/unbundle/as-indexed-file.js index 3e3ece669..969f0fbd0 100644 --- a/local-cli/bundle/output/unbundle/as-indexed-file.js +++ b/local-cli/bundle/output/unbundle/as-indexed-file.js @@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) { } = options; log('start'); - const {startupModules, lazyModules} = bundle.getUnbundle(); + const {startupModules, lazyModules, groups} = bundle.getUnbundle(); log('finish'); + const moduleGroups = ModuleGroups(groups, lazyModules); const startupCode = joinModules(startupModules); log('Writing unbundle output to:', bundleOutput); const writeUnbundle = writeBuffers( fs.createWriteStream(bundleOutput), - buildTableAndContents(startupCode, lazyModules, encoding) + buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding) ).then(() => log('Done writing unbundle output')); const sourceMap = - buildSourceMapWithMetaData({startupModules, lazyModules}); + buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups}); return Promise.all([ writeUnbundle, @@ -84,7 +85,7 @@ function entryOffset(n) { return (2 + n * 2) * SIZEOF_UINT32; } -function buildModuleTable(startupCode, buffers) { +function buildModuleTable(startupCode, buffers, moduleGroups) { // table format: // - num_entries: uint_32 number of entries // - startup_code_len: uint_32 length of the startup section @@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) { // - module_offset: uint_32 offset into the modules blob // - module_length: uint_32 length of the module code in bytes - const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0); + const moduleIds = Array.from(moduleGroups.modulesById.keys()); + const maxId = moduleIds.reduce((max, id) => Math.max(max, id)); const numEntries = maxId + 1; const table = new Buffer(entryOffset(numEntries)).fill(0); @@ -107,23 +109,50 @@ function buildModuleTable(startupCode, buffers) { // entries let codeOffset = startupCode.length; buffers.forEach(({id, buffer}) => { - const offset = entryOffset(id); - // module_offset - table.writeUInt32LE(codeOffset, offset); - // module_length - table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32); + const idsInGroup = moduleGroups.groups.has(id) + ? [id].concat(Array.from(moduleGroups.groups.get(id))) + : [id]; + + idsInGroup.forEach(moduleId => { + const offset = entryOffset(moduleId); + // module_offset + table.writeUInt32LE(codeOffset, offset); + // module_length + table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32); + }); codeOffset += buffer.length; }); return table; } -function buildModuleBuffers(modules, encoding) { - return modules.map( - module => moduleToBuffer(module.id, module.code, encoding)); +function groupCode(rootCode, moduleGroup, modulesById) { + if (!moduleGroup || !moduleGroup.size) { + return rootCode; + } + const code = [rootCode]; + for (const id of moduleGroup) { + code.push(modulesById.get(id).code); + } + + return code.join('\n'); } -function buildTableAndContents(startupCode, modules, encoding) { +function buildModuleBuffers(modules, moduleGroups, encoding) { + return modules + .filter(m => !moduleGroups.modulesInGroups.has(m.id)) + .map(({id, code}) => moduleToBuffer( + id, + groupCode( + code, + moduleGroups.groups.get(id), + moduleGroups.modulesById, + ), + encoding + )); +} + +function buildTableAndContents(startupCode, modules, moduleGroups, encoding) { // file contents layout: // - magic number char[4] 0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE) // - offset table table see `buildModuleTables` @@ -131,8 +160,8 @@ function buildTableAndContents(startupCode, modules, encoding) { // the startup code const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding); - const moduleBuffers = buildModuleBuffers(modules, encoding); - const table = buildModuleTable(startupCodeBuffer, moduleBuffers); + const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding); + const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups); return [ fileHeader, @@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) { ].concat(moduleBuffers.map(({buffer}) => buffer)); } +function ModuleGroups(groups, modules) { + return { + groups, + modulesById: new Map(modules.map(m => [m.id, m])), + modulesInGroups: new Set(concat(groups.values())), + }; +} + +function * concat(iterators) { + for (const it of iterators) { + yield * it; + } +} + module.exports = saveAsIndexedFile; diff --git a/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js b/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js index ba973736e..0d69657c8 100644 --- a/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js +++ b/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js @@ -10,13 +10,14 @@ const {combineSourceMaps, joinModules} = require('./util'); -module.exports = ({startupModules, lazyModules}) => { +module.exports = ({startupModules, lazyModules, moduleGroups}) => { const startupModule = { code: joinModules(startupModules), map: combineSourceMaps({modules: startupModules}), }; return combineSourceMaps({ modules: [startupModule].concat(lazyModules), + moduleGroups, withCustomOffsets: true, }); }; diff --git a/local-cli/bundle/output/unbundle/util.js b/local-cli/bundle/output/unbundle/util.js index d4504dfbd..16458bae1 100644 --- a/local-cli/bundle/output/unbundle/util.js +++ b/local-cli/bundle/output/unbundle/util.js @@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1; const Section = (line, column, map) => ({map, offset: {line, column}}); -function combineSourceMaps({modules, withCustomOffsets}) { +function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) { let offsets; const sections = []; const sourceMap = { @@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) { let line = 0; modules.forEach(({code, id, map, name}) => { - const hasOffset = withCustomOffsets && id != null; - const column = hasOffset ? wrapperEnd(code) : 0; + let column = 0; + let hasOffset = false; + let group; + let groupLines = 0; + + if (withCustomOffsets) { + if (moduleGroups && moduleGroups.modulesInGroups.has(id)) { + // this is a module appended to another module + return; + } + + if (moduleGroups && moduleGroups.groups.has(id)) { + group = moduleGroups.groups.get(id); + const otherModules = Array.from(group).map( + moduleId => moduleGroups.modulesById.get(moduleId)); + otherModules.forEach(m => { + groupLines += countLines(m.code); + }); + map = combineSourceMaps({ + modules: [{code, id, map, name}].concat(otherModules), + }); + } + + hasOffset = id != null; + column = wrapperEnd(code); + } + sections.push(Section(line, column, map || lineToLineSourceMap(code, name))); if (hasOffset) { offsets[id] = line; + for (const moduleId of group || []) { + offsets[moduleId] = line; + } } - line += countLines(code); + line += countLines(code) + groupLines; }); return sourceMap; diff --git a/packager/react-packager/src/Bundler/Bundle.js b/packager/react-packager/src/Bundler/Bundle.js index acca6649a..7bed098d1 100644 --- a/packager/react-packager/src/Bundler/Bundle.js +++ b/packager/react-packager/src/Bundler/Bundle.js @@ -17,7 +17,7 @@ const crypto = require('crypto'); const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL='; class Bundle extends BundleBase { - constructor({sourceMapUrl, dev, minify} = {}) { + constructor({sourceMapUrl, dev, minify, ramGroups} = {}) { super(); this._sourceMap = false; this._sourceMapUrl = sourceMapUrl; @@ -26,6 +26,7 @@ class Bundle extends BundleBase { this._dev = dev; this._minify = minify; + this._ramGroups = ramGroups; this._ramBundle = null; // cached RAM Bundle } @@ -111,9 +112,17 @@ class Bundle extends BundleBase { // separate modules we need to preload from the ones we don't const [startupModules, lazyModules] = partition(modules, shouldPreload); + const ramGroups = this._ramGroups; + let groups; this._ramBundle = { startupModules, lazyModules, + get groups() { + if (!groups) { + groups = createGroups(ramGroups || [], lazyModules); + } + return groups; + } }; } @@ -265,6 +274,10 @@ class Bundle extends BundleBase { ].join('\n'); } + setRamGroups(ramGroups) { + this._ramGroups = ramGroups; + } + toJSON() { this.assertFinalized('Cannot serialize bundle unless finalized'); @@ -318,4 +331,89 @@ function partition(array, predicate) { return [included, excluded]; } +function * filter(iterator, predicate) { + for (const value of iterator) { + if (predicate(value)) { + yield value; + } + } +} + +function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) { + seen.add(moduleTransport.id); + for (const [, {path}] of moduleTransport.meta.dependencyPairs) { + const dependency = moduleTransportsByPath.get(path); + if (dependency && !seen.has(dependency.id)) { + yield dependency.id; + yield * subtree(dependency, moduleTransportsByPath, seen); + } + } +} + +class ArrayMap extends Map { + get(key) { + let array = super.get(key); + if (!array) { + array = []; + this.set(key, array); + } + return array; + } +} + +function createGroups(ramGroups, lazyModules) { + // build two maps that allow to lookup module data + // by path or (numeric) module id; + const byPath = new Map(); + const byId = new Map(); + lazyModules.forEach(m => { + byPath.set(m.sourcePath, m); + byId.set(m.id, m.sourcePath); + }); + + // build a map of group root IDs to an array of module IDs in the group + const result = new Map( + ramGroups + .map(modulePath => { + const root = byPath.get(modulePath); + if (!root) { + throw Error(`Group root ${modulePath} is not part of the bundle`); + } + return [ + root.id, + // `subtree` yields the IDs of all transitive dependencies of a module + new Set(subtree(byPath.get(root.sourcePath), byPath)), + ]; + }) + ); + + if (ramGroups.length > 1) { + // build a map of all grouped module IDs to an array of group root IDs + const all = new ArrayMap(); + for (const [parent, children] of result) { + for (const module of children) { + all.get(module).push(parent); + } + } + + // find all module IDs that are part of more than one group + const doubles = filter(all, ([, parents]) => parents.length > 1); + for (const [moduleId, parents] of doubles) { + // remove them from their groups + parents.forEach(p => result.get(p).delete(moduleId)); + + // print a warning for each removed module + const parentNames = parents.map(byId.get, byId); + const lastName = parentNames.pop(); + console.warn( + `Module ${byId.get(moduleId)} belongs to groups ${ + parentNames.join(', ')}, and ${lastName + }. Removing it from all groups.` + ); + } + } + + return result; +} + module.exports = Bundle; diff --git a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js index 1f6f68ffb..a043c164f 100644 --- a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js +++ b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js @@ -333,6 +333,114 @@ describe('Bundle', () => { expect(deserialized.getMainModuleId()).toEqual(id); }); }); + + describe('random access bundle groups:', () => { + let moduleTransports; + beforeEach(() => { + moduleTransports = [ + transport('Product1', ['React', 'Relay']), + transport('React', ['ReactFoo', 'ReactBar']), + transport('ReactFoo', ['invariant']), + transport('invariant', []), + transport('ReactBar', ['cx']), + transport('cx', []), + transport('OtherFramework', ['OtherFrameworkFoo', 'OtherFrameworkBar']), + transport('OtherFrameworkFoo', ['invariant']), + transport('OtherFrameworkBar', ['crc32']), + transport('crc32', ['OtherFrameworkBar']), + ]; + }); + + it('can create a single group', () => { + bundle = createBundle([fsLocation('React')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('React'), new Set(['ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))], + ])); + }); + + it('can create two groups', () => { + bundle = createBundle([fsLocation('ReactFoo'), fsLocation('ReactBar')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('ReactFoo'), new Set([idFor('invariant')])], + [idFor('ReactBar'), new Set([idFor('cx')])], + ])); + }); + + it('can handle circular dependencies', () => { + bundle = createBundle([fsLocation('OtherFramework')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([[ + idFor('OtherFramework'), + new Set(['OtherFrameworkFoo', 'invariant', 'OtherFrameworkBar', 'crc32'].map(idFor)), + ]])); + }); + + it('omits modules that are contained by more than one group', () => { + bundle = createBundle([fsLocation('React'), fsLocation('OtherFramework')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([ + [idFor('React'), + new Set(['ReactFoo', 'ReactBar', 'cx'].map(idFor))], + [idFor('OtherFramework'), + new Set(['OtherFrameworkFoo', 'OtherFrameworkBar', 'crc32'].map(idFor))], + ])); + }); + + it('ignores missing dependencies', () => { + bundle = createBundle([fsLocation('Product1')]); + const {groups} = bundle.getUnbundle(); + expect(groups).toEqual(new Map([[ + idFor('Product1'), + new Set(['React', 'ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor)) + ]])); + }); + + it('throws for group roots that do not exist', () => { + bundle = createBundle([fsLocation('DoesNotExist')]); + expect(() => { + const {groups} = bundle.getUnbundle(); //eslint-disable-line no-unused-vars + }).toThrow(new Error(`Group root ${fsLocation('DoesNotExist')} is not part of the bundle`)); + }); + + function idFor(name) { + const {map} = idFor; + if (!map) { + idFor.map = new Map([[name, 0]]); + idFor.next = 1; + return 0; + } + + if (map.has(name)) { + return map.get(name); + } + + const id = idFor.next++; + map.set(name, id); + return id; + } + function createBundle(ramGroups, options = {}) { + const b = new Bundle(Object.assign(options, {ramGroups})); + moduleTransports.forEach(t => addModule({bundle: b, ...t})); + b.finalize(); + return b; + } + function fsLocation(name) { + return `/fs/${name}.js`; + } + function module(name) { + return {path: fsLocation(name)}; + } + function transport(name, deps) { + return createModuleTransport({ + name, + id: idFor(name), + sourcePath: fsLocation(name), + meta: {dependencyPairs: deps.map(d => [d, module(d)])}, + }); + } + }); }); @@ -375,7 +483,7 @@ function resolverFor(code, map) { }; } -function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) { +function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) { return bundle.addModule( resolverFor(code, map), null, @@ -384,7 +492,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill code, sourceCode, sourcePath, + id, map, + meta, virtual, polyfill, }), @@ -394,9 +504,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill function createModuleTransport(data) { return new ModuleTransport({ code: '', - id: '', sourceCode: '', sourcePath: '', + id: 'id' in data ? data.id : '', ...data, }); } diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index 2d640dae3..dfa7680f0 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -393,6 +393,8 @@ class Bundler { } return Promise.resolve(resolutionResponse).then(response => { + bundle.setRamGroups(response.transformOptions.transform.ramGroups); + Activity.endEvent(findEventId); onResolutionResponse(response);