Create an `OutputFn` that can build indexed RAM bundles

Summary: Adds functionality to assemble an indexed source map to the new Buck integration. This implementation supports startup section optimisations. Hooking it up, and grouping optimisations will be in follow-ups.

Reviewed By: jeanlauliac

Differential Revision: D5106985

fbshipit-source-id: cc4c6ac8cfe4e718fc8bb2a8a93cb88914c92e0b

packages/metro-bundler/src/ModuleGraph/output/__tests__/as-indexed-ram-bundle-test.js

@@ -0,0 +1,231 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+declare var jest: any;
+jest.disableAutomock();
+
+const indexedRamBundle = require('../indexed-ram-bundle');
+
+declare var describe: any;
+declare var expect: any;
+declare var it: (string, () => ?Promise<any>) => void;
+declare var beforeAll: (() => ?Promise<any>) => void;
+
+let code: Buffer;
+let map;
+let ids, modules, requireCall;
+const idForPath = ({path}) => getId(path);
+beforeAll(() => {
+  modules = [
+    makeModule('a', 'script'),
+    makeModule('b'),
+    makeModule('c'),
+    makeModule('d'),
+    makeModule('e'),
+    makeModule('f'),
+  ];
+  requireCall = makeModule('r', 'script', 'require(1);');
+
+  ids = new Map(modules.map(({file}, i) => [file.path, i]));
+  ({code, map} = createRamBundle());
+});
+
+it('starts the bundle file with the magic number', () => {
+  expect(code.readUInt32LE(0)).toBe(0xFB0BD1E5);
+});
+
+it('contains the number of modules in the module table', () => {
+  expect(code.readUInt32LE(SIZEOF_INT32)).toBe(modules.length);
+});
+
+it('has the length correct of the startup section', () => {
+  expect(code.readUInt32LE(SIZEOF_INT32 * 2))
+    .toBe(requireCall.file.code.length + 1);
+});
+
+it('contains the code after the offset table', () => {
+  const {codeOffset, startupSectionLength, table} = parseOffsetTable(code);
+
+  const startupSection =
+    code.slice(codeOffset, codeOffset + startupSectionLength - 1);
+  expect(startupSection.toString()).toBe(requireCall.file.code);
+
+  table.forEach(([offset, length], i) => {
+    const moduleCode =
+      code.slice(codeOffset + offset, codeOffset + offset + length - 1);
+    expect(moduleCode.toString()).toBe(modules[i].file.code);
+  });
+});
+
+it('creates a source map', () => {
+  let line = countLines(requireCall);
+  expect(map.sections.slice(1)).toEqual(modules.map(m => {
+    const section = {
+      map: m.file.map || lineByLineMap(m.file.path),
+      offset: {column: 0, line},
+    };
+    line += countLines(m);
+    return section;
+  }));
+  expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]);
+});
+
+describe('Optimization:', () => {
+  let last, preloaded;
+  beforeAll(() => {
+    last = modules[modules.length - 1];
+    preloaded = [modules[2], modules[3], last];
+    ({code, map} = createRamBundle(new Set(preloaded.map(getPath))));
+  });
+
+  it('supports additional modules in the startup section', () => {
+    const {codeOffset, startupSectionLength, table} = parseOffsetTable(code);
+
+    const startupSection =
+      code.slice(codeOffset, codeOffset + startupSectionLength - 1);
+    expect(startupSection.toString())
+      .toBe(preloaded.concat([requireCall]).map(getCode).join('\n'));
+
+
+    preloaded.forEach(m => {
+      const idx = idForPath(m.file);
+      expect(table[idx]).toEqual(m === last ? undefined : [0, 0]);
+    });
+
+    table.forEach(([offset, length], i) => {
+      if (offset !== 0 && length !== 0) {
+        const moduleCode =
+          code.slice(codeOffset + offset, codeOffset + offset + length - 1);
+        expect(moduleCode.toString()).toBe(modules[i].file.code);
+      }
+    });
+  });
+
+  it('reflects additional sources in the startup section in the source map', () => {
+    let line = preloaded.reduce(
+      (l, m) => l + countLines(m),
+      countLines(requireCall),
+    );
+
+    expect(map.x_facebook_offsets).toEqual([4, 5,,, 6]); // eslint-disable-line no-sparse-arrays
+
+    expect(map.sections.slice(1)).toEqual(
+      modules
+        .filter(not(Set.prototype.has), new Set(preloaded))
+        .map(m => {
+          const section = {
+            map: m.file.map || lineByLineMap(m.file.path),
+            offset: {column: 0, line},
+          };
+          line += countLines(m);
+          return section;
+        }
+    ));
+
+  });
+});
+
+function createRamBundle(preloadedModules = new Set()) {
+  const build = indexedRamBundle.createBuilder(preloadedModules);
+  const result = build({
+    filename: 'arbitrary/filename.js',
+    idForPath,
+    modules,
+    requireCalls: [requireCall],
+  });
+
+  if (typeof result.code === 'string') {
+    throw new Error('Expected a buffer, not a string');
+  }
+  return {code: result.code, map: result.map};
+}
+
+function makeModule(name, type = 'module', moduleCode = `var ${name};`) {
+  const path = `/${name}.js`;
+  return {
+    dependencies: [],
+    file: {
+      code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode,
+      map: type !== 'module'
+        ? null
+        : makeModuleMap(name, path),
+      path,
+      type,
+    },
+  };
+}
+
+function makeModuleMap(name, path) {
+  return {
+    version: 3,
+    mappings: Array(parseInt(name, 36) + 1).join(','),
+    names: [name],
+    sources: [path],
+  };
+}
+
+function makeModuleCode(moduleCode) {
+  return `__d(() => {${moduleCode}})`;
+}
+
+function getId(path) {
+  if (path === requireCall.file.path) {
+    return -1;
+  }
+
+  const id = ids.get(path);
+  if (id == null) {
+    throw new Error(`Unknown file: ${path}`);
+  }
+  return id;
+}
+
+function getCode(module) {
+  return module.file.code;
+}
+
+function getPath(module) {
+  return module.file.path;
+}
+
+const SIZEOF_INT32 = 4;
+function parseOffsetTable(buffer) {
+  const n = buffer.readUInt32LE(SIZEOF_INT32);
+  const startupSectionLength = buffer.readUInt32LE(SIZEOF_INT32 * 2);
+  const baseOffset = SIZEOF_INT32 * 3;
+  const table = Array(n);
+  for (let i = 0; i < n; ++i) {
+    const offset = baseOffset + i * 2 * SIZEOF_INT32;
+    table[i] = [buffer.readUInt32LE(offset), buffer.readUInt32LE(offset + SIZEOF_INT32)];
+  }
+  return {
+    codeOffset: baseOffset + n * 2 * SIZEOF_INT32,
+    startupSectionLength,
+    table,
+  };
+}
+
+function countLines(module) {
+  return module.file.code.split('\n').length;
+}
+
+function lineByLineMap(file) {
+  return {
+    file: file,
+    mappings: 'AAAA;',
+    names: [],
+    sources: [file],
+    version: 3,
+  };
+}
+
+const not = fn => function() { return !fn.apply(this, arguments); };

packages/metro-bundler/src/ModuleGraph/output/indexed-ram-bundle.js

@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+const buildSourceMapWithMetaData = require('../../../../local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js');
+
+const {buildTableAndContents, createModuleGroups} = require('../../../../local-cli/bundle/output/unbundle/as-indexed-file');
+const {concat} = require('./util');
+
+import type {FBIndexMap} from '../../lib/SourceMap.js';
+import type {OutputFn} from '../types.flow';
+
+function asIndexedRamBundle({
+  filename,
+  idForPath,
+  modules,
+  preloadedModules,
+  requireCalls,
+}) {
+  const [startup, deferred] = partition(modules, preloadedModules);
+  const startupModules = Array.from(concat(startup, requireCalls));
+  const deferredModules = deferred.map(m => toModuleTransport(m.file, idForPath));
+  const moduleGroups = createModuleGroups(new Map(), deferredModules);
+
+  const tableAndContents = buildTableAndContents(
+    startupModules.map(getModuleCode).join('\n'),
+    deferredModules,
+    moduleGroups,
+    'utf8',
+  );
+
+  return {
+    code: Buffer.concat(tableAndContents),
+    map: buildSourceMapWithMetaData({
+      fixWrapperOffset: false,
+      startupModules: startupModules.map(m => toModuleTransport(m.file, idForPath)),
+      lazyModules: deferredModules,
+    }),
+  };
+}
+
+function toModuleTransport(file, idForPath) {
+  return {
+    code: file.code,
+    id: idForPath(file),
+    map: file.map,
+    name: file.path,
+    sourcePath: file.path,
+  };
+}
+
+function getModuleCode(module) {
+  return module.file.code;
+}
+
+function partition(modules, preloadedModules) {
+  const startup = [];
+  const deferred = [];
+  for (const module of modules) {
+    (preloadedModules.has(module.file.path) ? startup : deferred).push(module);
+  }
+
+  return [startup, deferred];
+}
+
+function createBuilder(preloadedModules: Set<string>): OutputFn<FBIndexMap> {
+  return x => asIndexedRamBundle({preloadedModules, ...x});
+}
+exports.createBuilder = createBuilder;

packages/metro-bundler/src/ModuleGraph/output/plain-bundle.js

@@ -13,7 +13,7 @@
 const meta = require('../../../../local-cli/bundle/output/meta');
 
 const {createIndexMap} = require('./source-map');
-const {addModuleIdsToModuleWrapper} = require('./util');
+const {addModuleIdsToModuleWrapper, concat} = require('./util');
 
 import type {OutputFn} from '../types.flow';
 
@@ -55,16 +55,10 @@ function asPlainBundle({
   };
 }
 
-module.exports = (asPlainBundle: OutputFn);
+module.exports = (asPlainBundle: OutputFn<>);
 
 const reLine = /^/gm;
 function countLines(string: string): number {
   //$FlowFixMe This regular expression always matches
   return string.match(reLine).length;
 }
-
-function* concat<T>(...iterables: Array<Iterable<T>>): Iterable<T> {
-  for (const it of iterables) {
-    yield* it;
-  }
-}

packages/metro-bundler/src/ModuleGraph/output/util.js

@@ -44,6 +44,14 @@ exports.addModuleIdsToModuleWrapper = (
   );
 };
 
+exports.concat = function* concat<T>(
+  ...iterables: Array<Iterable<T>>
+): Iterable<T> {
+  for (const it of iterables) {
+    yield* it;
+  }
+};
+
 // Creates an idempotent function that returns numeric IDs for objects based
 // on their `path` property.
 exports.createIdForPathFn = (): ({path: string} => number) => {

packages/metro-bundler/src/ModuleGraph/types.flow.js

@@ -10,7 +10,7 @@
  */
 'use strict';
 
-import type {MappingsMap, SourceMap} from '../lib/SourceMap';
+import type {FBSourceMap, MappingsMap, SourceMap} from '../lib/SourceMap';
 import type {Ast} from 'babel-core';
 import type {Console} from 'console';
 export type {Transformer} from '../JSTransformer/worker/worker.js';
@@ -26,7 +26,7 @@ type Dependency = {|
 
 export type File = {|
   code: string,
-  map?: ?Object,
+  map?: ?MappingsMap,
   path: string,
   type: CodeFileTypes,
 |};
@@ -75,18 +75,18 @@ export type PostProcessModules = (
   entryPoints: Array<string>,
 ) => Iterable<Module>;
 
-export type OutputFn = ({|
+export type OutputFn<M: FBSourceMap | SourceMap = FBSourceMap | SourceMap> = ({|
   filename: string,
   idForPath: IdForPathFn,
   modules: Iterable<Module>,
   requireCalls: Iterable<Module>,
   sourceMapPath?: string,
-|}) => OutputResult;
+|}) => OutputResult<M>;
 
-type OutputResult = {|
+type OutputResult<M: FBSourceMap | SourceMap> = {|
   code: string | Buffer,
   extraFiles?: Iterable<[string, string | Buffer]>,
-  map: SourceMap,
+  map: M,
 |};
 
 export type PackageData = {|

packages/metro-bundler/src/lib/SourceMap.js

@@ -28,8 +28,9 @@ export type IndexMap = {
   version: number,
 };
 
+export type FBIndexMap = IndexMap & FBExtensions;
 export type SourceMap = IndexMap | MappingsMap;
-export type FBSourceMap = (IndexMap & FBExtensions) | (MappingsMap & FBExtensions);
+export type FBSourceMap = FBIndexMap | (MappingsMap & FBExtensions);
 
 function isMappingsMap(map: SourceMap)/*: %checks*/ {
   return map.mappings !== undefined;