Add unbundling to packager

Reviewed By: tadeuzagallo

Differential Revision: D2707409

fb-gh-sync-id: 30216c36066dae68d83622dba2d598e9dc0a29db

local-cli/bundle/bundle.js

@@ -17,10 +17,18 @@ const outputPrepack = require('./output/prepack');
 /**
  * Builds the bundle starting to look for dependencies at the given entry path.
  */
-function bundle(argv, config) {
+function bundleWithOutput(argv, config, output) {
   const args = parseCommandLine(bundleCommandLineArgs, argv);
-  const output = args.prepack ? outputPrepack : outputBundle;
+  if (!output) {
+    output = args.prepack ? outputPrepack : outputBundle;
+  }
   return buildBundle(args, config, output);
+
+}
+
+function bundle(argv, config) {
+  return bundleWithOutput(argv, config);
 }
 
 module.exports = bundle;
+module.exports.withOutput = bundleWithOutput;

local-cli/bundle/output/bundle.js

@@ -33,7 +33,7 @@ function saveBundleAndMap(bundle, options, log) {
     'bundle-encoding': encoding,
     dev,
     'sourcemap-output': sourcemapOutput,
-} = options;
+  } = options;
 
   log('start');
   const codeWithMap = createCodeWithMap(bundle, dev);

local-cli/bundle/output/unbundle.js

@@ -0,0 +1,126 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const fs = require('fs');
+const Promise = require('promise');
+const writeFile = require('./writeFile');
+
+const MAGIC_STARTUP_MODULE_ID = '';
+
+function buildBundle(packagerClient, requestOptions) {
+  return packagerClient.buildBundle({...requestOptions, unbundle: true});
+}
+
+function saveUnbundle(bundle, options, log) {
+  const {
+    'bundle-output': bundleOutput,
+    'bundle-encoding': encoding,
+    dev,
+    'sourcemap-output': sourcemapOutput,
+  } = options;
+
+  log('start');
+  const {startupCode, modules} = bundle.getUnbundle({minify: !dev});
+  log('finish');
+
+  log('Writing unbundle output to:', bundleOutput);
+  const writeUnbundle = writeBuffers(
+    fs.createWriteStream(bundleOutput),
+    buildTableAndContents(startupCode, modules, encoding)
+  );
+
+  writeUnbundle.then(() => log('Done writing unbundle output'));
+
+  if (sourcemapOutput) {
+    log('Writing sourcemap output to:', sourcemapOutput);
+    const writeMap = writeFile(sourcemapOutput, '', null);
+    writeMap.then(() => log('Done writing sourcemap output'));
+    return Promise.all([writeUnbundle, writeMap]);
+  } else {
+    return writeUnbundle;
+  }
+}
+
+/* global Buffer: true */
+const nullByteBuffer = Buffer(1).fill(0);
+
+const moduleToBuffer = ({name, code}, encoding) => ({
+  name,
+  buffer: Buffer.concat([
+    Buffer(code, encoding),
+    nullByteBuffer // create \0-terminated strings
+  ])
+});
+
+function buildModuleBuffers(startupCode, modules, encoding) {
+  return (
+    [moduleToBuffer({name: '', code: startupCode}, encoding)]
+      .concat(modules.map(module => moduleToBuffer(module, encoding)))
+  );
+}
+
+function uInt32Buffer(n) {
+  const buffer = Buffer(4);
+  buffer.writeUInt32LE(n, 0); // let's assume LE for now :)
+  return buffer;
+}
+
+function buildModuleTable(buffers) {
+  // table format:
+  //  - table_length: uint_32 length of all table entries in bytes
+  //  - entries: entry...
+  //
+  // entry:
+  //  - module_id: NUL terminated utf8 string
+  //  - module_offset: uint_32 offset into the module string
+  //  - module_length: uint_32 length of the module string, including terminating NUL byte
+
+  const numBuffers = buffers.length;
+
+  const tableLengthBuffer = uInt32Buffer(0);
+  let tableLength = 4; // the table length itself, 4 == tableLengthBuffer.length
+  let currentOffset = 0;
+
+  const offsetTable = [tableLengthBuffer];
+  for (let i = 0; i < numBuffers; i++) {
+    const {name, buffer: {length}} = buffers[i];
+    const entry = Buffer.concat([
+      Buffer(i === 0 ? MAGIC_STARTUP_MODULE_ID : name, 'utf8'),
+      nullByteBuffer,
+      uInt32Buffer(currentOffset),
+      uInt32Buffer(length)
+    ]);
+    currentOffset += length;
+    tableLength += entry.length;
+    offsetTable.push(entry);
+  }
+
+  tableLengthBuffer.writeUInt32LE(tableLength, 0);
+  return Buffer.concat(offsetTable);
+}
+
+function buildTableAndContents(startupCode, modules, encoding) {
+  const buffers = buildModuleBuffers(startupCode, modules, encoding);
+  const table = buildModuleTable(buffers, encoding);
+  return [table].concat(buffers.map(({buffer}) => buffer));
+}
+
+function writeBuffers(stream, buffers) {
+  buffers.forEach(buffer => stream.write(buffer));
+  return new Promise((resolve, reject) => {
+    stream.on('error', reject);
+    stream.on('finish', () => resolve());
+    stream.end();
+  });
+}
+
+exports.build = buildBundle;
+exports.save = saveUnbundle;
+exports.formatName = 'bundle';

local-cli/bundle/unbundle.js

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const bundleWithOutput = require('./bundle').withOutput;
+const outputUnbundle = require('./output/unbundle');
+
+/**
+ * Builds the bundle starting to look for dependencies at the given entry path.
+ */
+function unbundle(argv, config) {
+  return bundleWithOutput(argv, config, outputUnbundle);
+}
+
+module.exports = unbundle;

local-cli/cli.js

@@ -28,6 +28,7 @@ var runAndroid = require('./runAndroid/runAndroid');
 var server = require('./server/server');
 var TerminalAdapter = require('yeoman-environment/lib/adapter.js');
 var yeoman = require('yeoman-environment');
+var unbundle = require('./bundle/unbundle');
 var upgrade = require('./upgrade/upgrade');
 
 var fs = require('fs');
@@ -40,6 +41,7 @@ gracefulFs.gracefulify(fs);
 var documentedCommands = {
   'start': [server, 'starts the webserver'],
   'bundle': [bundle, 'builds the javascript bundle for offline use'],
+  'unbundle': [unbundle, 'builds javascript as "unbundle" for offline use'],
   'new-library': [library, 'generates a native library bridge'],
   'link': [link, 'Adds a third-party library to your project. Example: react-native link awesome-camera'],
   'android': [generateWrapper, 'generates an Android project for your app'],

packager/react-packager/src/Bundler/Bundle.js

@@ -16,6 +16,14 @@ const Activity = require('../Activity');
 
 const SOURCEMAPPING_URL = '\n\/\/@ sourceMappingURL=';
 
+const minifyCode = code =>
+  UglifyJS.minify(code, {fromString: true, ascii_only: true}).code;
+const getCode = x => x.code;
+const getMinifiedCode = x => minifyCode(x.code);
+const getNameAndCode = ({name, code}) => ({name, code});
+const getNameAndMinifiedCode =
+  ({name, code}) => ({name, code: minifyCode(code)});
+
 class Bundle {
   constructor(sourceMapUrl) {
     this._finalized = false;
@@ -24,6 +32,8 @@ class Bundle {
     this._sourceMap = false;
     this._sourceMapUrl = sourceMapUrl;
     this._shouldCombineSourceMaps = false;
+    this._numPrependedModules = 0;
+    this._numRequireCalls = 0;
   }
 
   setMainModuleId(moduleId) {
@@ -48,6 +58,10 @@ class Bundle {
     return this._modules;
   }
 
+  setNumPrependedModules(n) {
+    this._numPrependedModules = n;
+  }
+
   addAsset(asset) {
     this._assets.push(asset);
   }
@@ -76,6 +90,7 @@ class Bundle {
       sourceCode: code,
       sourcePath: name + '.js',
     }));
+    this._numRequireCalls += 1;
   }
 
   _assertFinalized() {
@@ -141,6 +156,26 @@ class Bundle {
     return source;
   }
 
+  getUnbundle({minify}) {
+    const allModules = this._modules.slice();
+    const prependedModules = this._numPrependedModules;
+    const requireCalls = this._numRequireCalls;
+
+    const modules =
+      allModules
+        .splice(prependedModules, allModules.length - requireCalls - prependedModules);
+    const startupCode =
+      allModules
+        .map(minify ? getMinifiedCode : getCode)
+        .join('\n');
+
+    return {
+      startupCode,
+      modules:
+        modules.map(minify ? getNameAndMinifiedCode : getNameAndCode)
+    };
+  }
+
   getMinifiedSourceAndMap(dev) {
     this._assertFinalized();
 
@@ -336,6 +371,8 @@ class Bundle {
       assets: this._assets,
       sourceMapUrl: this._sourceMapUrl,
       mainModuleId: this._mainModuleId,
+      numPrependedModules: this._numPrependedModules,
+      numRequireCalls: this._numRequireCalls,
     };
   }
 
@@ -345,6 +382,8 @@ class Bundle {
     bundle._assets = json.assets;
     bundle._modules = json.modules;
     bundle._sourceMapUrl = json.sourceMapUrl;
+    bundle._numPrependedModules = json.numPrependedModules;
+    bundle._numRequireCalls = json.numRequireCalls;
 
     Object.freeze(bundle._modules);
     Object.seal(bundle._modules);

packager/react-packager/src/Bundler/__tests__/Bundler-test.js

@@ -130,7 +130,10 @@ describe('Bundler', function() {
       });
 
     wrapModule.mockImpl(function(response, module, code) {
-      return Promise.resolve('lol ' + code + ' lol');
+      return module.getName().then(name => ({
+        name,
+        code: 'lol ' + code + ' lol'
+      }));
     });
 
     sizeOf.mockImpl(function(path, cb) {
@@ -160,6 +163,7 @@ describe('Bundler', function() {
       sourceMapUrl: 'source_map_url',
     }).then(function(p) {
         expect(p.addModule.mock.calls[0][0]).toEqual({
+          name: 'foo',
           code: 'lol transformed /root/foo.js lol',
           map: 'sourcemap /root/foo.js',
           sourceCode: 'source /root/foo.js',
@@ -167,6 +171,7 @@ describe('Bundler', function() {
         });
 
         expect(p.addModule.mock.calls[1][0]).toEqual({
+          name: 'bar',
           code: 'lol transformed /root/bar.js lol',
           map: 'sourcemap /root/bar.js',
           sourceCode: 'source /root/bar.js',
@@ -183,6 +188,7 @@ describe('Bundler', function() {
         };
 
         expect(p.addModule.mock.calls[2][0]).toEqual({
+          name: 'image!img',
           code: 'lol module.exports = ' +
             JSON.stringify(imgModule_DEPRECATED) +
             '; lol',
@@ -212,6 +218,7 @@ describe('Bundler', function() {
         };
 
         expect(p.addModule.mock.calls[3][0]).toEqual({
+          name: 'new_image.png',
           code: 'lol module.exports = require("AssetRegistry").registerAsset(' +
             JSON.stringify(imgModule) +
             '); lol',
@@ -224,6 +231,7 @@ describe('Bundler', function() {
         });
 
         expect(p.addModule.mock.calls[4][0]).toEqual({
+          name: 'package/file.json',
           code: 'lol module.exports = {"json":true}; lol',
           sourceCode: 'module.exports = {"json":true};',
           sourcePath: '/root/file.json',

packager/react-packager/src/Bundler/index.js

@@ -140,6 +140,7 @@ class Bundler {
     sourceMapUrl,
     dev: isDev,
     platform,
+    unbundle: isUnbundle,
   }) {
     // Const cannot have the same name as the method (babel/babel#2834)
     const bbundle = new Bundle(sourceMapUrl);
@@ -147,7 +148,7 @@ class Bundler {
     let transformEventId;
 
     const moduleSystem = this._resolver.getModuleSystemDependencies(
-      { dev: isDev, platform }
+      { dev: isDev, platform, isUnbundle }
     );
 
     return this.getDependencies(entryFile, isDev, platform).then((response) => {
@@ -168,6 +169,8 @@ class Bundler {
       }
 
       bbundle.setMainModuleId(response.mainModuleId);
+      bbundle.setNumPrependedModules(
+        response.numPrependedDependencies + moduleSystem.length);
       return Promise.all(
         dependencies.map(
           module => this._transformModule(
@@ -317,8 +320,9 @@ class Bundler {
       module,
       transformed.code
     ).then(
-      code => new ModuleTransport({
-        code: code,
+      ({code, name}) => new ModuleTransport({
+        code,
+        name,
         map: transformed.map,
         sourceCode: transformed.sourceCode,
         sourcePath: transformed.sourcePath,

packager/react-packager/src/DependencyResolver/DependencyGraph/ResolutionResponse.js

@@ -14,6 +14,7 @@ class ResolutionResponse {
     this.asyncDependencies = [];
     this.mainModuleId = null;
     this.mocks = null;
+    this.numPrependedDependencies = 0;
     this._mappings = Object.create(null);
     this._finalized = false;
   }
@@ -50,6 +51,7 @@ class ResolutionResponse {
   prependDependency(module) {
     this._assertNotFinalized();
     this.dependencies.unshift(module);
+    this.numPrependedDependencies += 1;
   }
 
   pushAsyncDependency(dependency) {

packager/react-packager/src/Resolver/__tests__/Resolver-test.js

@@ -627,7 +627,8 @@ describe('Resolver', function() {
         createModule('test module', ['x', 'y']),
         code
       ).then(processedCode => {
-        expect(processedCode).toEqual([
+        expect(processedCode.name).toEqual('test module');
+        expect(processedCode.code).toEqual([
           '__d(\'test module\',function(global, require,' +
             ' module, exports) {  ' +
             // single line import

packager/react-packager/src/Resolver/index.js

@@ -60,6 +60,10 @@ const getDependenciesValidateOpts = declareOpts({
     type: 'string',
     required: false,
   },
+  isUnbundle: {
+    type: 'boolean',
+    default: false
+  },
 });
 
 class Resolver {
@@ -115,7 +119,9 @@ class Resolver {
         ? path.join(__dirname, 'polyfills/prelude_dev.js')
         : path.join(__dirname, 'polyfills/prelude.js');
 
-    const moduleSystem = path.join(__dirname, 'polyfills/require.js');
+    const moduleSystem = opts.isUnbundle
+        ? path.join(__dirname, 'polyfills/require-unbundle.js')
+        : path.join(__dirname, 'polyfills/require.js');
 
     return [
       prelude,
@@ -152,7 +158,7 @@ class Resolver {
   wrapModule(resolutionResponse, module, code) {
     return Promise.resolve().then(() => {
       if (module.isPolyfill()) {
-        return Promise.resolve(code);
+        return Promise.resolve({code});
       }
 
       const resolvedDeps = Object.create(null);
@@ -179,14 +185,13 @@ class Resolver {
           }
         };
 
-        return module.getName().then(
-          name => defineModuleCode({
-            code: code.replace(replacePatterns.IMPORT_RE, relativizeCode)
-                      .replace(replacePatterns.EXPORT_RE, relativizeCode)
-                      .replace(replacePatterns.REQUIRE_RE, relativizeCode),
-            moduleName: name,
-          })
-        );
+        code = code
+          .replace(replacePatterns.IMPORT_RE, relativizeCode)
+          .replace(replacePatterns.EXPORT_RE, relativizeCode)
+          .replace(replacePatterns.REQUIRE_RE, relativizeCode);
+
+        return module.getName().then(name =>
+          ({name, code: defineModuleCode(name, code)}));
       });
     });
   }
@@ -197,7 +202,7 @@ class Resolver {
 
 }
 
-function defineModuleCode({moduleName, code}) {
+function defineModuleCode(moduleName, code) {
   return [
     `__d(`,
     `'${moduleName}',`,

packager/react-packager/src/Resolver/polyfills/require-unbundle.js

@@ -0,0 +1,73 @@
+'use strict';
+
+((global) => {
+  const {ErrorUtils, __nativeRequire} = global;
+  global.require = require;
+  global.__d = define;
+
+  const modules = Object.create(null);
+
+  const loadModule = ErrorUtils ?
+    guardedLoadModule : loadModuleImplementation;
+
+  function define(moduleId, factory) {
+    modules[moduleId] = {
+      factory,
+      hasError: false,
+      exports: undefined,
+    };
+  }
+
+  function require(moduleId) {
+    const module = modules[moduleId];
+    return module && module.exports || loadModule(moduleId, module);
+  }
+
+  function guardedLoadModule(moduleId, module) {
+    try {
+      return loadModuleImplementation(moduleId, module);
+    } catch (e) {
+      ErrorUtils.reportFatalError(e);
+    }
+  }
+
+  function loadModuleImplementation(moduleId, module) {
+    if (!module) {
+      __nativeRequire(moduleId);
+      module = modules[moduleId];
+    }
+
+    if (!module) {
+      throw unknownModuleError(moduleId);
+    }
+
+    if (module.hasError) {
+      throw moduleThrewError(moduleId);
+    }
+
+    const exports = module.exports = {};
+    const {factory} = module;
+    try {
+      const moduleObject = {exports};
+      factory(global, require, moduleObject, exports);
+      return (module.exports = moduleObject.exports);
+    } catch(e) {
+      module.hasError = true;
+      module.exports = undefined;
+    }
+  }
+
+  function unknownModuleError(id) {
+    let message = 'Requiring unknown module "' + id + '".';
+    if (__DEV__) {
+      message +=
+        'If you are sure the module is there, try restarting the packager.';
+    }
+    return Error(message);
+  }
+
+  function moduleThrewError(id) {
+    return Error('Requiring module "' + id + '", which threw an exception.');
+  }
+
+})(this);

packager/react-packager/src/Server/__tests__/Server-test.js

@@ -115,6 +115,7 @@ describe('processRequest', () => {
         dev: true,
         platform: undefined,
         runBeforeMainModule: ['InitializeJavaScriptAppEngine'],
+        unbundle: false,
       });
     });
   });
@@ -134,6 +135,7 @@ describe('processRequest', () => {
         dev: true,
         platform: 'ios',
         runBeforeMainModule: ['InitializeJavaScriptAppEngine'],
+        unbundle: false,
       });
     });
   });
@@ -274,6 +276,7 @@ describe('processRequest', () => {
           dev: true,
           platform: undefined,
           runBeforeMainModule: ['InitializeJavaScriptAppEngine'],
+          unbundle: false,
         })
       );
     });
@@ -292,6 +295,7 @@ describe('processRequest', () => {
             dev: false,
             platform: undefined,
             runBeforeMainModule: ['InitializeJavaScriptAppEngine'],
+            unbundle: false,
           })
         );
     });

packager/react-packager/src/Server/index.js

@@ -102,6 +102,10 @@ const bundleOpts = declareOpts({
       'InitializeJavaScriptAppEngine'
     ],
   },
+  unbundle: {
+    type: 'boolean',
+    default: false,
+  }
 });
 
 const dependencyOpts = declareOpts({

packager/react-packager/src/lib/ModuleTransport.js

@@ -9,6 +9,8 @@
 'use strict';
 
 function ModuleTransport(data) {
+  this.name = data.name;
+
   assertExists(data, 'code');
   this.code = data.code;