Adding support for RAM bundles - multiple files version

Reviewed By: davidaurelio

Differential Revision: D5277684

fbshipit-source-id: 77041375026d5c186022c8108acace7b519902fb

packages/metro-bundler/src/ModuleGraph/output/__tests__/multiple-files-ram-bundle-test.js

@@ -0,0 +1,166 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+declare var jest: any;
+
+const multipleFilesRamBundle = require('../multiple-files-ram-bundle');
+
+const {addModuleIdsToModuleWrapper} = require('../util');
+
+declare var describe: any;
+declare var expect: any;
+declare var it: (string, () => ?Promise<any>) => void;
+declare var beforeAll: (() => ?Promise<any>) => void;
+
+let code: Buffer;
+let map;
+let extraFiles;
+let ids, modules, requireCall;
+const idForPath = ({path}) => getId(path);
+
+beforeAll(() => {
+  modules = [
+    makeModule('a', [], 'script'),
+    makeModule('b'),
+    makeModule('c', ['f']),
+    makeModule('d', ['e']),
+    makeModule('e', ['c']),
+    makeModule('f'),
+  ];
+  requireCall = makeModule('r', [], 'script', 'require(1);');
+  ids = new Map(modules.map(({file}, i) => [file.path, i]));
+  ({code, extraFiles, map} = createRamBundle());
+});
+
+it('does not start the bundle file with the magic number (not a binary one)', () => {
+  expect(new Buffer(code).readUInt32LE(0)).not.toBe(0xFB0BD1E5);
+});
+
+it('contains the startup code on the main file', () => {
+  expect(code.toString()).toBe('require(1);');
+});
+
+it('creates a source map', () => {
+  let line = countLines(requireCall);
+  expect(map.sections.slice(1)).toEqual(modules.map(m => {
+    const section = {
+      map: m.file.map || lineByLineMap(m.file.path),
+      offset: {column: 0, line},
+    };
+    line += countLines(m);
+    return section;
+  }));
+  expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]);
+});
+
+it('creates a magic file with the number', () => {
+  expect(extraFiles).toBeDefined();
+  // $FlowFixMe "extraFiles" is always defined at this point.
+  expect(extraFiles.get('UNBUNDLE')).toBeDefined();
+  // $FlowFixMe "extraFiles" is always defined at this point.
+  expect(extraFiles.get('UNBUNDLE').readUInt32LE(0)).toBe(0xFB0BD1E5);
+});
+
+it('bundles each file separately', () => {
+  expect(extraFiles).toBeDefined();
+
+  modules.forEach((module, i) => {
+    // $FlowFixMe "extraFiles" is always defined at this point.
+    expect(extraFiles.get(`js-modules/${i}.js`).toString())
+      .toBe(expectedCode(modules[i]));
+  });
+});
+
+function createRamBundle(preloadedModules = new Set(), ramGroups) {
+  const build = multipleFilesRamBundle.createBuilder(preloadedModules, ramGroups);
+  const result = build({
+    filename: 'arbitrary/filename.js',
+    idForPath,
+    modules,
+    requireCalls: [requireCall],
+  });
+
+  return {code: result.code, map: result.map, extraFiles: result.extraFiles};
+}
+
+function makeModule(name, deps = [], type = 'module', moduleCode = `var ${name};`) {
+  const path = makeModulePath(name);
+  return {
+    dependencies: deps.map(makeDependency),
+    file: {
+      code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode,
+      map: type !== 'module'
+        ? null
+        : makeModuleMap(name, path),
+      path,
+      type,
+    },
+  };
+}
+
+function makeModuleMap(name, path) {
+  return {
+    version: 3,
+    mappings: Array(parseInt(name, 36) + 1).join(','),
+    names: [name],
+    sources: [path],
+  };
+}
+
+function makeModuleCode(moduleCode) {
+  return `__d(() => {${moduleCode}})`;
+}
+
+function makeModulePath(name) {
+  return `/${name}.js`;
+}
+
+function makeDependency(name) {
+  const path = makeModulePath(name);
+  return {
+    id: name,
+    path,
+  };
+}
+
+function expectedCode(module) {
+  const {file} = module;
+  return file.type === 'module'
+    ? addModuleIdsToModuleWrapper(module, idForPath)
+    : file.code;
+}
+
+function getId(path) {
+  if (path === requireCall.file.path) {
+    return -1;
+  }
+
+  const id = ids.get(path);
+  if (id == null) {
+    throw new Error(`Unknown file: ${path}`);
+  }
+  return id;
+}
+
+function countLines(module) {
+  return module.file.code.split('\n').length;
+}
+
+function lineByLineMap(file) {
+  return {
+    file,
+    mappings: 'AAAA;',
+    names: [],
+    sources: [file],
+    version: 3,
+  };
+}

packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js

@@ -15,7 +15,7 @@ const nullthrows = require('fbjs/lib/nullthrows');
 
 const {createRamBundleGroups} = require('../../Bundler/util');
 const {buildTableAndContents, createModuleGroups} = require('../../shared/output/unbundle/as-indexed-file');
-const {addModuleIdsToModuleWrapper, concat} = require('./util');
+const {concat, getModuleCode, partition, toModuleTransport} = require('./util');
 
 import type {FBIndexMap} from '../../lib/SourceMap.js';
 import type {OutputFn} from '../types.flow';
@@ -52,35 +52,6 @@ function asIndexedRamBundle({
   };
 }
 
-function toModuleTransport(module, idForPath) {
-  const {dependencies, file} = module;
-  return {
-    code: getModuleCode(module, idForPath),
-    dependencies,
-    id: idForPath(file),
-    map: file.map,
-    name: file.path,
-    sourcePath: file.path,
-  };
-}
-
-function getModuleCode(module, idForPath) {
-  const {file} = module;
-  return file.type === 'module'
-    ? addModuleIdsToModuleWrapper(module, idForPath)
-    : file.code;
-}
-
-function partition(modules, preloadedModules) {
-  const startup = [];
-  const deferred = [];
-  for (const module of modules) {
-    (preloadedModules.has(module.file.path) ? startup : deferred).push(module);
-  }
-
-  return [startup, deferred];
-}
-
 function *subtree(
   moduleTransport,
   moduleTransportsByPath,

packages/metro-bundler/src/ModuleGraph/output/multiple-files-ram-bundle.js

@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+const MAGIC_UNBUNDLE_NUMBER = require('../../shared/output/unbundle/magic-number');
+const MAGIC_UNBUNDLE_FILENAME = 'UNBUNDLE';
+const JS_MODULES = 'js-modules';
+
+const buildSourceMapWithMetaData = require('../../shared/output/unbundle/build-unbundle-sourcemap-with-metadata.js');
+const path = require('path');
+
+const {concat, getModuleCode, partition, toModuleTransport} = require('./util');
+
+import type {FBIndexMap} from '../../lib/SourceMap.js';
+import type {OutputFn} from '../types.flow';
+
+function asMultipleFilesRamBundle({
+  filename,
+  idForPath,
+  modules,
+  requireCalls,
+  preloadedModules,
+}) {
+  const [startup, deferred] = partition(modules, preloadedModules);
+  const startupModules = Array.from(concat(startup, requireCalls));
+  const deferredModules = deferred.map(m => toModuleTransport(m, idForPath));
+  const magicFileContents = new Buffer(4);
+
+  // Just concatenate all startup modules, one after the other.
+  const code = startupModules.map(m => getModuleCode(m, idForPath)).join('\n');
+
+  // Write one file per module, wrapped with __d() call if it proceeds.
+  const extraFiles = new Map();
+  deferredModules.forEach(deferredModule => {
+    extraFiles.set(
+      path.join(JS_MODULES, deferredModule.id + '.js'),
+      new Buffer(deferredModule.code),
+    );
+  });
+
+  // Prepare and write magic number file.
+  magicFileContents.writeUInt32LE(MAGIC_UNBUNDLE_NUMBER, 0);
+  extraFiles.set(MAGIC_UNBUNDLE_FILENAME, magicFileContents);
+
+  // Create the source map (with no module groups, as they are ignored).
+  const map = buildSourceMapWithMetaData({
+    fixWrapperOffset: false,
+    lazyModules: deferredModules,
+    moduleGroups: null,
+    startupModules: startupModules.map(m => toModuleTransport(m, idForPath)),
+  });
+
+  return {code, extraFiles, map};
+}
+
+function createBuilder(
+  preloadedModules: Set<string>,
+  ramGroupHeads: ?$ReadOnlyArray<string>,
+): OutputFn<FBIndexMap> {
+  return x => asMultipleFilesRamBundle({...x, preloadedModules, ramGroupHeads});
+}
+
+exports.createBuilder = createBuilder;

packages/metro-bundler/src/ModuleGraph/output/util.js

@@ -21,10 +21,10 @@ import type {IdForPathFn, Module} from '../types.flow';
 //
 // This function adds the numeric module ID, and an array with dependencies of
 // the dependencies of the module before the closing parenthesis.
-exports.addModuleIdsToModuleWrapper = (
+function addModuleIdsToModuleWrapper(
   module: Module,
   idForPath: {path: string} => number,
-): string => {
+): string {
   const {dependencies, file} = module;
   const {code} = file;
   const index = code.lastIndexOf(')');
@@ -44,8 +44,25 @@ exports.addModuleIdsToModuleWrapper = (
     depencyIds +
     code.slice(index)
   );
-};
+}
 
+exports.addModuleIdsToModuleWrapper = addModuleIdsToModuleWrapper;
+
+// Adds the module ids to a file if the file is a module. If it's not (e.g. a
+// script) it just keeps it as-is.
+function getModuleCode(
+  module: Module,
+  idForPath: IdForPathFn,
+) {
+  const {file} = module;
+  return file.type === 'module'
+    ? addModuleIdsToModuleWrapper(module, idForPath)
+    : file.code;
+}
+
+exports.getModuleCode = getModuleCode;
+
+// Concatenates many iterables, by calling them sequentially.
 exports.concat = function* concat<T>(
   ...iterables: Array<Iterable<T>>
 ): Iterable<T> {
@@ -79,3 +96,35 @@ exports.requireCallsTo = function* (
     yield virtualModule(`require(${idForPath(module.file)});`);
   }
 };
+
+// Divides the modules into two types: the ones that are loaded at startup, and
+// the ones loaded deferredly (lazy loaded).
+exports.partition = (
+  modules: Iterable<Module>,
+  preloadedModules: Set<string>,
+): Array<Array<Module>> => {
+  const startup = [];
+  const deferred = [];
+  for (const module of modules) {
+    (preloadedModules.has(module.file.path) ? startup : deferred).push(module);
+  }
+
+  return [startup, deferred];
+};
+
+// Transforms a new Module object into an old one, so that it can be passed
+// around code.
+exports.toModuleTransport = (
+  module: Module,
+  idForPath: IdForPathFn,
+) => {
+  const {dependencies, file} = module;
+  return {
+    code: getModuleCode(module, idForPath),
+    dependencies,
+    id: idForPath(file),
+    map: file.map,
+    name: file.path,
+    sourcePath: file.path,
+  };
+};

packages/metro-bundler/src/node-haste/__tests__/Module-test.js

@@ -14,7 +14,6 @@ jest.mock('fs')
   .mock('../DependencyGraph/DependencyGraphHelpers')
   .mock('../../lib/TransformCaching');
 
-console.log(require.resolve('../../lib/TransformCaching'));
 const Module = require('../Module');
 const ModuleCache = require('../ModuleCache');
 const DependencyGraphHelpers = require('../DependencyGraph/DependencyGraphHelpers');

packages/metro-bundler/src/shared/output/unbundle/as-indexed-file.js

@@ -151,7 +151,7 @@ function groupCode(rootCode, moduleGroup, modulesById) {
   }
   const code = [rootCode];
   for (const id of moduleGroup) {
-    code.push((modulesById.get(id) || {}).code);
+    code.push((modulesById.get(id) || {code: ''}).code);
   }
 
   return code.join('\n');