Add support for module groups to iOS Random Access Bundle format

Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8
This commit is contained in:
David Aurelio 2016-09-16 10:03:34 -07:00 committed by Facebook Github Bot 5
parent 9ff4d31dc4
commit b62ed2f291
6 changed files with 306 additions and 24 deletions

View File

@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) {
} = options;
log('start');
const {startupModules, lazyModules} = bundle.getUnbundle();
const {startupModules, lazyModules, groups} = bundle.getUnbundle();
log('finish');
const moduleGroups = ModuleGroups(groups, lazyModules);
const startupCode = joinModules(startupModules);
log('Writing unbundle output to:', bundleOutput);
const writeUnbundle = writeBuffers(
fs.createWriteStream(bundleOutput),
buildTableAndContents(startupCode, lazyModules, encoding)
buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding)
).then(() => log('Done writing unbundle output'));
const sourceMap =
buildSourceMapWithMetaData({startupModules, lazyModules});
buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups});
return Promise.all([
writeUnbundle,
@ -84,7 +85,7 @@ function entryOffset(n) {
return (2 + n * 2) * SIZEOF_UINT32;
}
function buildModuleTable(startupCode, buffers) {
function buildModuleTable(startupCode, buffers, moduleGroups) {
// table format:
// - num_entries: uint_32 number of entries
// - startup_code_len: uint_32 length of the startup section
@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) {
// - module_offset: uint_32 offset into the modules blob
// - module_length: uint_32 length of the module code in bytes
const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0);
const moduleIds = Array.from(moduleGroups.modulesById.keys());
const maxId = moduleIds.reduce((max, id) => Math.max(max, id));
const numEntries = maxId + 1;
const table = new Buffer(entryOffset(numEntries)).fill(0);
@ -107,23 +109,50 @@ function buildModuleTable(startupCode, buffers) {
// entries
let codeOffset = startupCode.length;
buffers.forEach(({id, buffer}) => {
const offset = entryOffset(id);
// module_offset
table.writeUInt32LE(codeOffset, offset);
// module_length
table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
const idsInGroup = moduleGroups.groups.has(id)
? [id].concat(Array.from(moduleGroups.groups.get(id)))
: [id];
idsInGroup.forEach(moduleId => {
const offset = entryOffset(moduleId);
// module_offset
table.writeUInt32LE(codeOffset, offset);
// module_length
table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
});
codeOffset += buffer.length;
});
return table;
}
function buildModuleBuffers(modules, encoding) {
return modules.map(
module => moduleToBuffer(module.id, module.code, encoding));
function groupCode(rootCode, moduleGroup, modulesById) {
if (!moduleGroup || !moduleGroup.size) {
return rootCode;
}
const code = [rootCode];
for (const id of moduleGroup) {
code.push(modulesById.get(id).code);
}
return code.join('\n');
}
function buildTableAndContents(startupCode, modules, encoding) {
function buildModuleBuffers(modules, moduleGroups, encoding) {
return modules
.filter(m => !moduleGroups.modulesInGroups.has(m.id))
.map(({id, code}) => moduleToBuffer(
id,
groupCode(
code,
moduleGroups.groups.get(id),
moduleGroups.modulesById,
),
encoding
));
}
function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
// file contents layout:
// - magic number char[4] 0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
// - offset table table see `buildModuleTables`
@ -131,8 +160,8 @@ function buildTableAndContents(startupCode, modules, encoding) {
// the startup code
const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding);
const moduleBuffers = buildModuleBuffers(modules, encoding);
const table = buildModuleTable(startupCodeBuffer, moduleBuffers);
const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding);
const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups);
return [
fileHeader,
@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) {
].concat(moduleBuffers.map(({buffer}) => buffer));
}
function ModuleGroups(groups, modules) {
return {
groups,
modulesById: new Map(modules.map(m => [m.id, m])),
modulesInGroups: new Set(concat(groups.values())),
};
}
function * concat(iterators) {
for (const it of iterators) {
yield * it;
}
}
module.exports = saveAsIndexedFile;

View File

@ -10,13 +10,14 @@
const {combineSourceMaps, joinModules} = require('./util');
module.exports = ({startupModules, lazyModules}) => {
module.exports = ({startupModules, lazyModules, moduleGroups}) => {
const startupModule = {
code: joinModules(startupModules),
map: combineSourceMaps({modules: startupModules}),
};
return combineSourceMaps({
modules: [startupModule].concat(lazyModules),
moduleGroups,
withCustomOffsets: true,
});
};

View File

@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1;
const Section = (line, column, map) => ({map, offset: {line, column}});
function combineSourceMaps({modules, withCustomOffsets}) {
function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) {
let offsets;
const sections = [];
const sourceMap = {
@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) {
let line = 0;
modules.forEach(({code, id, map, name}) => {
const hasOffset = withCustomOffsets && id != null;
const column = hasOffset ? wrapperEnd(code) : 0;
let column = 0;
let hasOffset = false;
let group;
let groupLines = 0;
if (withCustomOffsets) {
if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
// this is a module appended to another module
return;
}
if (moduleGroups && moduleGroups.groups.has(id)) {
group = moduleGroups.groups.get(id);
const otherModules = Array.from(group).map(
moduleId => moduleGroups.modulesById.get(moduleId));
otherModules.forEach(m => {
groupLines += countLines(m.code);
});
map = combineSourceMaps({
modules: [{code, id, map, name}].concat(otherModules),
});
}
hasOffset = id != null;
column = wrapperEnd(code);
}
sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
if (hasOffset) {
offsets[id] = line;
for (const moduleId of group || []) {
offsets[moduleId] = line;
}
}
line += countLines(code);
line += countLines(code) + groupLines;
});
return sourceMap;

View File

@ -17,7 +17,7 @@ const crypto = require('crypto');
const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
class Bundle extends BundleBase {
constructor({sourceMapUrl, dev, minify} = {}) {
constructor({sourceMapUrl, dev, minify, ramGroups} = {}) {
super();
this._sourceMap = false;
this._sourceMapUrl = sourceMapUrl;
@ -26,6 +26,7 @@ class Bundle extends BundleBase {
this._dev = dev;
this._minify = minify;
this._ramGroups = ramGroups;
this._ramBundle = null; // cached RAM Bundle
}
@ -111,9 +112,17 @@ class Bundle extends BundleBase {
// separate modules we need to preload from the ones we don't
const [startupModules, lazyModules] = partition(modules, shouldPreload);
const ramGroups = this._ramGroups;
let groups;
this._ramBundle = {
startupModules,
lazyModules,
get groups() {
if (!groups) {
groups = createGroups(ramGroups || [], lazyModules);
}
return groups;
}
};
}
@ -265,6 +274,10 @@ class Bundle extends BundleBase {
].join('\n');
}
setRamGroups(ramGroups) {
this._ramGroups = ramGroups;
}
toJSON() {
this.assertFinalized('Cannot serialize bundle unless finalized');
@ -318,4 +331,89 @@ function partition(array, predicate) {
return [included, excluded];
}
function * filter(iterator, predicate) {
for (const value of iterator) {
if (predicate(value)) {
yield value;
}
}
}
function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) {
seen.add(moduleTransport.id);
for (const [, {path}] of moduleTransport.meta.dependencyPairs) {
const dependency = moduleTransportsByPath.get(path);
if (dependency && !seen.has(dependency.id)) {
yield dependency.id;
yield * subtree(dependency, moduleTransportsByPath, seen);
}
}
}
class ArrayMap extends Map {
get(key) {
let array = super.get(key);
if (!array) {
array = [];
this.set(key, array);
}
return array;
}
}
function createGroups(ramGroups, lazyModules) {
// build two maps that allow to lookup module data
// by path or (numeric) module id;
const byPath = new Map();
const byId = new Map();
lazyModules.forEach(m => {
byPath.set(m.sourcePath, m);
byId.set(m.id, m.sourcePath);
});
// build a map of group root IDs to an array of module IDs in the group
const result = new Map(
ramGroups
.map(modulePath => {
const root = byPath.get(modulePath);
if (!root) {
throw Error(`Group root ${modulePath} is not part of the bundle`);
}
return [
root.id,
// `subtree` yields the IDs of all transitive dependencies of a module
new Set(subtree(byPath.get(root.sourcePath), byPath)),
];
})
);
if (ramGroups.length > 1) {
// build a map of all grouped module IDs to an array of group root IDs
const all = new ArrayMap();
for (const [parent, children] of result) {
for (const module of children) {
all.get(module).push(parent);
}
}
// find all module IDs that are part of more than one group
const doubles = filter(all, ([, parents]) => parents.length > 1);
for (const [moduleId, parents] of doubles) {
// remove them from their groups
parents.forEach(p => result.get(p).delete(moduleId));
// print a warning for each removed module
const parentNames = parents.map(byId.get, byId);
const lastName = parentNames.pop();
console.warn(
`Module ${byId.get(moduleId)} belongs to groups ${
parentNames.join(', ')}, and ${lastName
}. Removing it from all groups.`
);
}
}
return result;
}
module.exports = Bundle;

View File

@ -333,6 +333,114 @@ describe('Bundle', () => {
expect(deserialized.getMainModuleId()).toEqual(id);
});
});
describe('random access bundle groups:', () => {
let moduleTransports;
beforeEach(() => {
moduleTransports = [
transport('Product1', ['React', 'Relay']),
transport('React', ['ReactFoo', 'ReactBar']),
transport('ReactFoo', ['invariant']),
transport('invariant', []),
transport('ReactBar', ['cx']),
transport('cx', []),
transport('OtherFramework', ['OtherFrameworkFoo', 'OtherFrameworkBar']),
transport('OtherFrameworkFoo', ['invariant']),
transport('OtherFrameworkBar', ['crc32']),
transport('crc32', ['OtherFrameworkBar']),
];
});
it('can create a single group', () => {
bundle = createBundle([fsLocation('React')]);
const {groups} = bundle.getUnbundle();
expect(groups).toEqual(new Map([
[idFor('React'), new Set(['ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))],
]));
});
it('can create two groups', () => {
bundle = createBundle([fsLocation('ReactFoo'), fsLocation('ReactBar')]);
const {groups} = bundle.getUnbundle();
expect(groups).toEqual(new Map([
[idFor('ReactFoo'), new Set([idFor('invariant')])],
[idFor('ReactBar'), new Set([idFor('cx')])],
]));
});
it('can handle circular dependencies', () => {
bundle = createBundle([fsLocation('OtherFramework')]);
const {groups} = bundle.getUnbundle();
expect(groups).toEqual(new Map([[
idFor('OtherFramework'),
new Set(['OtherFrameworkFoo', 'invariant', 'OtherFrameworkBar', 'crc32'].map(idFor)),
]]));
});
it('omits modules that are contained by more than one group', () => {
bundle = createBundle([fsLocation('React'), fsLocation('OtherFramework')]);
const {groups} = bundle.getUnbundle();
expect(groups).toEqual(new Map([
[idFor('React'),
new Set(['ReactFoo', 'ReactBar', 'cx'].map(idFor))],
[idFor('OtherFramework'),
new Set(['OtherFrameworkFoo', 'OtherFrameworkBar', 'crc32'].map(idFor))],
]));
});
it('ignores missing dependencies', () => {
bundle = createBundle([fsLocation('Product1')]);
const {groups} = bundle.getUnbundle();
expect(groups).toEqual(new Map([[
idFor('Product1'),
new Set(['React', 'ReactFoo', 'invariant', 'ReactBar', 'cx'].map(idFor))
]]));
});
it('throws for group roots that do not exist', () => {
bundle = createBundle([fsLocation('DoesNotExist')]);
expect(() => {
const {groups} = bundle.getUnbundle(); //eslint-disable-line no-unused-vars
}).toThrow(new Error(`Group root ${fsLocation('DoesNotExist')} is not part of the bundle`));
});
function idFor(name) {
const {map} = idFor;
if (!map) {
idFor.map = new Map([[name, 0]]);
idFor.next = 1;
return 0;
}
if (map.has(name)) {
return map.get(name);
}
const id = idFor.next++;
map.set(name, id);
return id;
}
function createBundle(ramGroups, options = {}) {
const b = new Bundle(Object.assign(options, {ramGroups}));
moduleTransports.forEach(t => addModule({bundle: b, ...t}));
b.finalize();
return b;
}
function fsLocation(name) {
return `/fs/${name}.js`;
}
function module(name) {
return {path: fsLocation(name)};
}
function transport(name, deps) {
return createModuleTransport({
name,
id: idFor(name),
sourcePath: fsLocation(name),
meta: {dependencyPairs: deps.map(d => [d, module(d)])},
});
}
});
});
@ -375,7 +483,7 @@ function resolverFor(code, map) {
};
}
function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) {
function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) {
return bundle.addModule(
resolverFor(code, map),
null,
@ -384,7 +492,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
code,
sourceCode,
sourcePath,
id,
map,
meta,
virtual,
polyfill,
}),
@ -394,9 +504,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill
function createModuleTransport(data) {
return new ModuleTransport({
code: '',
id: '',
sourceCode: '',
sourcePath: '',
id: 'id' in data ? data.id : '',
...data,
});
}

View File

@ -393,6 +393,8 @@ class Bundler {
}
return Promise.resolve(resolutionResponse).then(response => {
bundle.setRamGroups(response.transformOptions.transform.ramGroups);
Activity.endEvent(findEventId);
onResolutionResponse(response);