From b4621838303eed62429208cf0bdc6518d6219ce6 Mon Sep 17 00:00:00 2001 From: Miguel Jimenez Esun Date: Tue, 20 Jun 2017 11:43:05 -0700 Subject: [PATCH] Adding support for RAM bundles - multiple files version Reviewed By: davidaurelio Differential Revision: D5277684 fbshipit-source-id: 77041375026d5c186022c8108acace7b519902fb --- .../multiple-files-ram-bundle-test.js | 166 ++++++++++++++++++ .../ModuleGraph/output/indexed-ram-bundle.js | 31 +--- .../output/multiple-files-ram-bundle.js | 71 ++++++++ .../src/ModuleGraph/output/util.js | 55 +++++- .../src/node-haste/__tests__/Module-test.js | 1 - .../shared/output/unbundle/as-indexed-file.js | 2 +- 6 files changed, 291 insertions(+), 35 deletions(-) create mode 100644 packages/metro-bundler/src/ModuleGraph/output/__tests__/multiple-files-ram-bundle-test.js create mode 100644 packages/metro-bundler/src/ModuleGraph/output/multiple-files-ram-bundle.js diff --git a/packages/metro-bundler/src/ModuleGraph/output/__tests__/multiple-files-ram-bundle-test.js b/packages/metro-bundler/src/ModuleGraph/output/__tests__/multiple-files-ram-bundle-test.js new file mode 100644 index 00000000..8b7705b3 --- /dev/null +++ b/packages/metro-bundler/src/ModuleGraph/output/__tests__/multiple-files-ram-bundle-test.js @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ +'use strict'; + +declare var jest: any; + +const multipleFilesRamBundle = require('../multiple-files-ram-bundle'); + +const {addModuleIdsToModuleWrapper} = require('../util'); + +declare var describe: any; +declare var expect: any; +declare var it: (string, () => ?Promise) => void; +declare var beforeAll: (() => ?Promise) => void; + +let code: Buffer; +let map; +let extraFiles; +let ids, modules, requireCall; +const idForPath = ({path}) => getId(path); + +beforeAll(() => { + modules = [ + makeModule('a', [], 'script'), + makeModule('b'), + makeModule('c', ['f']), + makeModule('d', ['e']), + makeModule('e', ['c']), + makeModule('f'), + ]; + requireCall = makeModule('r', [], 'script', 'require(1);'); + ids = new Map(modules.map(({file}, i) => [file.path, i])); + ({code, extraFiles, map} = createRamBundle()); +}); + +it('does not start the bundle file with the magic number (not a binary one)', () => { + expect(new Buffer(code).readUInt32LE(0)).not.toBe(0xFB0BD1E5); +}); + +it('contains the startup code on the main file', () => { + expect(code.toString()).toBe('require(1);'); +}); + +it('creates a source map', () => { + let line = countLines(requireCall); + expect(map.sections.slice(1)).toEqual(modules.map(m => { + const section = { + map: m.file.map || lineByLineMap(m.file.path), + offset: {column: 0, line}, + }; + line += countLines(m); + return section; + })); + expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]); +}); + +it('creates a magic file with the number', () => { + expect(extraFiles).toBeDefined(); + // $FlowFixMe "extraFiles" is always defined at this point. + expect(extraFiles.get('UNBUNDLE')).toBeDefined(); + // $FlowFixMe "extraFiles" is always defined at this point. + expect(extraFiles.get('UNBUNDLE').readUInt32LE(0)).toBe(0xFB0BD1E5); +}); + +it('bundles each file separately', () => { + expect(extraFiles).toBeDefined(); + + modules.forEach((module, i) => { + // $FlowFixMe "extraFiles" is always defined at this point. + expect(extraFiles.get(`js-modules/${i}.js`).toString()) + .toBe(expectedCode(modules[i])); + }); +}); + +function createRamBundle(preloadedModules = new Set(), ramGroups) { + const build = multipleFilesRamBundle.createBuilder(preloadedModules, ramGroups); + const result = build({ + filename: 'arbitrary/filename.js', + idForPath, + modules, + requireCalls: [requireCall], + }); + + return {code: result.code, map: result.map, extraFiles: result.extraFiles}; +} + +function makeModule(name, deps = [], type = 'module', moduleCode = `var ${name};`) { + const path = makeModulePath(name); + return { + dependencies: deps.map(makeDependency), + file: { + code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode, + map: type !== 'module' + ? null + : makeModuleMap(name, path), + path, + type, + }, + }; +} + +function makeModuleMap(name, path) { + return { + version: 3, + mappings: Array(parseInt(name, 36) + 1).join(','), + names: [name], + sources: [path], + }; +} + +function makeModuleCode(moduleCode) { + return `__d(() => {${moduleCode}})`; +} + +function makeModulePath(name) { + return `/${name}.js`; +} + +function makeDependency(name) { + const path = makeModulePath(name); + return { + id: name, + path, + }; +} + +function expectedCode(module) { + const {file} = module; + return file.type === 'module' + ? addModuleIdsToModuleWrapper(module, idForPath) + : file.code; +} + +function getId(path) { + if (path === requireCall.file.path) { + return -1; + } + + const id = ids.get(path); + if (id == null) { + throw new Error(`Unknown file: ${path}`); + } + return id; +} + +function countLines(module) { + return module.file.code.split('\n').length; +} + +function lineByLineMap(file) { + return { + file, + mappings: 'AAAA;', + names: [], + sources: [file], + version: 3, + }; +} diff --git a/packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js b/packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js index 286cc27b..f773d475 100644 --- a/packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js +++ b/packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js @@ -15,7 +15,7 @@ const nullthrows = require('fbjs/lib/nullthrows'); const {createRamBundleGroups} = require('../../Bundler/util'); const {buildTableAndContents, createModuleGroups} = require('../../shared/output/unbundle/as-indexed-file'); -const {addModuleIdsToModuleWrapper, concat} = require('./util'); +const {concat, getModuleCode, partition, toModuleTransport} = require('./util'); import type {FBIndexMap} from '../../lib/SourceMap.js'; import type {OutputFn} from '../types.flow'; @@ -52,35 +52,6 @@ function asIndexedRamBundle({ }; } -function toModuleTransport(module, idForPath) { - const {dependencies, file} = module; - return { - code: getModuleCode(module, idForPath), - dependencies, - id: idForPath(file), - map: file.map, - name: file.path, - sourcePath: file.path, - }; -} - -function getModuleCode(module, idForPath) { - const {file} = module; - return file.type === 'module' - ? addModuleIdsToModuleWrapper(module, idForPath) - : file.code; -} - -function partition(modules, preloadedModules) { - const startup = []; - const deferred = []; - for (const module of modules) { - (preloadedModules.has(module.file.path) ? startup : deferred).push(module); - } - - return [startup, deferred]; -} - function *subtree( moduleTransport, moduleTransportsByPath, diff --git a/packages/metro-bundler/src/ModuleGraph/output/multiple-files-ram-bundle.js b/packages/metro-bundler/src/ModuleGraph/output/multiple-files-ram-bundle.js new file mode 100644 index 00000000..55730a36 --- /dev/null +++ b/packages/metro-bundler/src/ModuleGraph/output/multiple-files-ram-bundle.js @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ +'use strict'; + +const MAGIC_UNBUNDLE_NUMBER = require('../../shared/output/unbundle/magic-number'); +const MAGIC_UNBUNDLE_FILENAME = 'UNBUNDLE'; +const JS_MODULES = 'js-modules'; + +const buildSourceMapWithMetaData = require('../../shared/output/unbundle/build-unbundle-sourcemap-with-metadata.js'); +const path = require('path'); + +const {concat, getModuleCode, partition, toModuleTransport} = require('./util'); + +import type {FBIndexMap} from '../../lib/SourceMap.js'; +import type {OutputFn} from '../types.flow'; + +function asMultipleFilesRamBundle({ + filename, + idForPath, + modules, + requireCalls, + preloadedModules, +}) { + const [startup, deferred] = partition(modules, preloadedModules); + const startupModules = Array.from(concat(startup, requireCalls)); + const deferredModules = deferred.map(m => toModuleTransport(m, idForPath)); + const magicFileContents = new Buffer(4); + + // Just concatenate all startup modules, one after the other. + const code = startupModules.map(m => getModuleCode(m, idForPath)).join('\n'); + + // Write one file per module, wrapped with __d() call if it proceeds. + const extraFiles = new Map(); + deferredModules.forEach(deferredModule => { + extraFiles.set( + path.join(JS_MODULES, deferredModule.id + '.js'), + new Buffer(deferredModule.code), + ); + }); + + // Prepare and write magic number file. + magicFileContents.writeUInt32LE(MAGIC_UNBUNDLE_NUMBER, 0); + extraFiles.set(MAGIC_UNBUNDLE_FILENAME, magicFileContents); + + // Create the source map (with no module groups, as they are ignored). + const map = buildSourceMapWithMetaData({ + fixWrapperOffset: false, + lazyModules: deferredModules, + moduleGroups: null, + startupModules: startupModules.map(m => toModuleTransport(m, idForPath)), + }); + + return {code, extraFiles, map}; +} + +function createBuilder( + preloadedModules: Set, + ramGroupHeads: ?$ReadOnlyArray, +): OutputFn { + return x => asMultipleFilesRamBundle({...x, preloadedModules, ramGroupHeads}); +} + +exports.createBuilder = createBuilder; diff --git a/packages/metro-bundler/src/ModuleGraph/output/util.js b/packages/metro-bundler/src/ModuleGraph/output/util.js index f9a652a7..8f2cc23c 100644 --- a/packages/metro-bundler/src/ModuleGraph/output/util.js +++ b/packages/metro-bundler/src/ModuleGraph/output/util.js @@ -21,10 +21,10 @@ import type {IdForPathFn, Module} from '../types.flow'; // // This function adds the numeric module ID, and an array with dependencies of // the dependencies of the module before the closing parenthesis. -exports.addModuleIdsToModuleWrapper = ( +function addModuleIdsToModuleWrapper( module: Module, idForPath: {path: string} => number, -): string => { +): string { const {dependencies, file} = module; const {code} = file; const index = code.lastIndexOf(')'); @@ -44,8 +44,25 @@ exports.addModuleIdsToModuleWrapper = ( depencyIds + code.slice(index) ); -}; +} +exports.addModuleIdsToModuleWrapper = addModuleIdsToModuleWrapper; + +// Adds the module ids to a file if the file is a module. If it's not (e.g. a +// script) it just keeps it as-is. +function getModuleCode( + module: Module, + idForPath: IdForPathFn, +) { + const {file} = module; + return file.type === 'module' + ? addModuleIdsToModuleWrapper(module, idForPath) + : file.code; +} + +exports.getModuleCode = getModuleCode; + +// Concatenates many iterables, by calling them sequentially. exports.concat = function* concat( ...iterables: Array> ): Iterable { @@ -79,3 +96,35 @@ exports.requireCallsTo = function* ( yield virtualModule(`require(${idForPath(module.file)});`); } }; + +// Divides the modules into two types: the ones that are loaded at startup, and +// the ones loaded deferredly (lazy loaded). +exports.partition = ( + modules: Iterable, + preloadedModules: Set, +): Array> => { + const startup = []; + const deferred = []; + for (const module of modules) { + (preloadedModules.has(module.file.path) ? startup : deferred).push(module); + } + + return [startup, deferred]; +}; + +// Transforms a new Module object into an old one, so that it can be passed +// around code. +exports.toModuleTransport = ( + module: Module, + idForPath: IdForPathFn, +) => { + const {dependencies, file} = module; + return { + code: getModuleCode(module, idForPath), + dependencies, + id: idForPath(file), + map: file.map, + name: file.path, + sourcePath: file.path, + }; +}; diff --git a/packages/metro-bundler/src/node-haste/__tests__/Module-test.js b/packages/metro-bundler/src/node-haste/__tests__/Module-test.js index bfc047d9..583beba6 100644 --- a/packages/metro-bundler/src/node-haste/__tests__/Module-test.js +++ b/packages/metro-bundler/src/node-haste/__tests__/Module-test.js @@ -14,7 +14,6 @@ jest.mock('fs') .mock('../DependencyGraph/DependencyGraphHelpers') .mock('../../lib/TransformCaching'); -console.log(require.resolve('../../lib/TransformCaching')); const Module = require('../Module'); const ModuleCache = require('../ModuleCache'); const DependencyGraphHelpers = require('../DependencyGraph/DependencyGraphHelpers'); diff --git a/packages/metro-bundler/src/shared/output/unbundle/as-indexed-file.js b/packages/metro-bundler/src/shared/output/unbundle/as-indexed-file.js index d636455b..5d24a1f3 100644 --- a/packages/metro-bundler/src/shared/output/unbundle/as-indexed-file.js +++ b/packages/metro-bundler/src/shared/output/unbundle/as-indexed-file.js @@ -151,7 +151,7 @@ function groupCode(rootCode, moduleGroup, modulesById) { } const code = [rootCode]; for (const id of moduleGroup) { - code.push((modulesById.get(id) || {}).code); + code.push((modulesById.get(id) || {code: ''}).code); } return code.join('\n');