Add infra for Prepack build option

Summary: This adds a build option for using Prepack (an experimental packager) to
build a bundle. It doesn't actually take on the npm package dependency
because it's not published/open source (yet).

This will be used while we experiment and should be maintained as the
build system changes so that we can continue getting fresh builds.

I found that saveBundleAndMap and processBundle were over abstracted and
got in my way so I inlined it and removed the unit tests because the unit
test was testing trivial code that is likely to change interface.

I went with a separate build phase and a separate Bundle class even though
there are a lot of commonalities. I imagine that the requirements for
Prepack will continue to diverge. Especially for source maps but a larger
refactor could try to unify these a bit more. The fact that modules are
wrapped before the write phase seems to be an unfortunate architecture
that makes this difficult.
Closes https://github.com/facebook/react-native/pull/4226

Reviewed By: amasad

Differential Revision: D2673760

Pulled By: sebmarkbage

fb-gh-sync-id: 299ccc42e4be1d9dee19ade443ea3388db2e39a8

local-cli/bundle/__tests__/saveBundleAndMap-test.js

@@ -1,63 +0,0 @@
-/**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- */
-'use strict';
-
-jest.autoMockOff();
-
-jest.mock('fs');
-jest.mock('../sign');
-
-const saveBundleAndMap = require('../saveBundleAndMap');
-const fs = require('fs');
-const temp = require('temp');
-
-const code = 'const foo = "bar";';
-const map = JSON.stringify({
-  version: 3,
-  file: 'foo.js.map',
-  sources: ['foo.js'],
-  sourceRoot: '/',
-  names: ['bar'],
-  mappings: 'AAA0B,kBAAhBA,QAAOC,SACjBD,OAAOC,OAAO'
-});
-
-describe('saveBundleAndMap', () => {
-  beforeEach(() => {
-    fs.writeFileSync = jest.genMockFn();
-  });
-
-  it('should save bundle', () => {
-    const codeWithMap = {code: code};
-    const bundleOutput = temp.path({suffix: '.bundle'});
-
-    saveBundleAndMap(
-      codeWithMap,
-      'ios',
-      bundleOutput,
-      'utf8',
-    );
-
-    expect(fs.writeFileSync.mock.calls[0]).toEqual([bundleOutput, code, 'utf8']);
-  });
-
-  it('should save sourcemaps if required so', () => {
-    const codeWithMap = {code: code, map: map};
-    const bundleOutput = temp.path({suffix: '.bundle'});
-    const sourceMapOutput = temp.path({suffix: '.map'});
-    saveBundleAndMap(
-      codeWithMap,
-      'ios',
-      bundleOutput,
-      'utf8',
-      sourceMapOutput
-    );
-
-    expect(fs.writeFileSync.mock.calls[1]).toEqual([sourceMapOutput, map]);
-  });
-});

local-cli/bundle/buildBundle.js

@@ -8,11 +8,57 @@
  */
 'use strict';
 
+const fs = require('fs');
 const log = require('../util/log').out('bundle');
-const processBundle = require('./processBundle');
 const Promise = require('promise');
 const ReactPackager = require('../../packager/react-packager');
-const saveBundleAndMap = require('./saveBundleAndMap');
+const saveAssets = require('./saveAssets');
+
+const sign = require('./sign');
+
+function saveBundleAndMap(
+  bundle,
+  bundleOutput,
+  encoding,
+  sourcemapOutput,
+  dev
+) {
+  log('start');
+  let codeWithMap;
+  if (!dev) {
+    codeWithMap = bundle.getMinifiedSourceAndMap(dev);
+  } else {
+    codeWithMap = {
+      code: bundle.getSource({ dev }),
+      map: JSON.stringify(bundle.getSourceMap({ dev })),
+    };
+  }
+  log('finish');
+
+  log('Writing bundle output to:', bundleOutput);
+  fs.writeFileSync(bundleOutput, sign(codeWithMap.code), encoding);
+  log('Done writing bundle output');
+
+  if (sourcemapOutput) {
+    log('Writing sourcemap output to:', sourcemapOutput);
+    fs.writeFileSync(sourcemapOutput, codeWithMap.map);
+    log('Done writing sourcemap output');
+  }
+}
+
+function savePrepackBundleAndMap(
+  bundle,
+  bundleOutput,
+  sourcemapOutput,
+  bridgeConfig
+) {
+  log('Writing prepack bundle output to:', bundleOutput);
+  const result = bundle.build({
+    batchedBridgeConfig: bridgeConfig
+  });
+  fs.writeFileSync(bundleOutput, result, 'ucs-2');
+  log('Done writing prepack bundle output');
+}
 
 function buildBundle(args, config) {
   return new Promise((resolve, reject) => {
@@ -36,24 +82,56 @@ function buildBundle(args, config) {
       platform: args.platform,
     };
 
-    resolve(ReactPackager.createClientFor(options).then(client => {
-      log('Created ReactPackager');
-      return client.buildBundle(requestOpts)
+    const prepack = args.prepack;
+
+    const client = ReactPackager.createClientFor(options);
+
+    client.then(() => log('Created ReactPackager'));
+
+    // Build and save the bundle
+    let bundle;
+    if (prepack) {
+      bundle = client.then(c => c.buildPrepackBundle(requestOpts))
         .then(outputBundle => {
-          log('Closing client');
-          client.close();
+          savePrepackBundleAndMap(
+            outputBundle,
+            args['bundle-output'],
+            args['sourcemap-output'],
+            args['bridge-config']
+          );
           return outputBundle;
-        })
-        .then(outputBundle => processBundle(outputBundle, args.dev))
-        .then(outputBundle => saveBundleAndMap(
-          outputBundle,
+        });
+    } else {
+      bundle = client.then(c => c.buildBundle(requestOpts))
+        .then(outputBundle => {
+          saveBundleAndMap(
+            outputBundle,
+            args['bundle-output'],
+            args['bundle-encoding'],
+            args['sourcemap-output'],
+            args.dev
+          );
+          return outputBundle;
+        });
+    }
+
+    // When we're done bundling, close the client
+    bundle.then(() => client.then(c => {
+      log('Closing client');
+      c.close();
+    }));
+
+    // Save the assets of the bundle
+    const assets = bundle
+        .then(outputBundle => outputBundle.getAssets())
+        .then(outputAssets => saveAssets(
+          outputAssets,
           args.platform,
-          args['bundle-output'],
-          args['bundle-encoding'],
-          args['sourcemap-output'],
           args['assets-dest']
         ));
-    }));
+
+    // When we're done saving the assets, we're done.
+    resolve(assets);
   });
 }
 

local-cli/bundle/bundleCommandLineArgs.js

@@ -27,6 +27,14 @@ module.exports = [
     command: 'dev',
     description: 'If false, warnings are disabled and the bundle is minified',
     default: true,
+  }, {
+    command: 'prepack',
+    description: 'If true, the output bundle will use the Prepack format.',
+    default: false
+  }, {
+    command: 'bridge-config',
+    description: 'File name of a a JSON export of __fbBatchedBridgeConfig. Used by Prepack. Ex. ./bridgeconfig.json',
+    type: 'string'
   }, {
     command: 'bundle-output',
     description: 'File name where to store the resulting bundle, ex. /tmp/groups.bundle',

local-cli/bundle/processBundle.js

@@ -1,29 +0,0 @@
-/**
- * Copyright (c) 2015-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- */
-'use strict';
-
-const log = require('../util/log').out('bundle');
-
-function processBundle(input, dev) {
-  log('start');
-  let bundle;
-  if (!dev) {
-    bundle = input.getMinifiedSourceAndMap(dev);
-  } else {
-    bundle = {
-      code: input.getSource({ dev }),
-      map: JSON.stringify(input.getSourceMap({ dev })),
-    };
-  }
-  bundle.assets = input.getAssets();
-  log('finish');
-  return bundle;
-}
-
-module.exports = processBundle;

local-cli/bundle/saveAssets.js

@@ -14,26 +14,12 @@ const getAssetDestPathAndroid = require('./getAssetDestPathAndroid');
 const getAssetDestPathIOS = require('./getAssetDestPathIOS');
 const log = require('../util/log').out('bundle');
 const path = require('path');
-const sign = require('./sign');
 
-function saveBundleAndMap(
-  codeWithMap,
+function saveAssets(
+  assets,
   platform,
-  bundleOutput,
-  encoding,
-  sourcemapOutput,
   assetsDest
 ) {
-  log('Writing bundle output to:', bundleOutput);
-  fs.writeFileSync(bundleOutput, sign(codeWithMap.code), encoding);
-  log('Done writing bundle output');
-
-  if (sourcemapOutput) {
-    log('Writing sourcemap output to:', sourcemapOutput);
-    fs.writeFileSync(sourcemapOutput, codeWithMap.map);
-    log('Done writing sourcemap output');
-  }
-
   if (!assetsDest) {
     console.warn('Assets destination folder is not set, skipping...');
     return Promise.resolve();
@@ -44,7 +30,7 @@ function saveBundleAndMap(
     : getAssetDestPathIOS;
 
   const filesToCopy = Object.create(null); // Map src -> dest
-  codeWithMap.assets
+  assets
     .filter(asset => !asset.deprecated)
     .forEach(asset =>
       asset.scales.forEach((scale, idx) => {
@@ -94,4 +80,4 @@ function copy(src, dest, callback) {
   });
 }
 
-module.exports = saveBundleAndMap;
+module.exports = saveAssets;

packager/react-packager/index.js

@@ -35,6 +35,15 @@ exports.buildBundle = function(options, bundleOptions) {
     });
 };
 
+exports.buildPrepackBundle = function(options, bundleOptions) {
+  var server = createNonPersistentServer(options);
+  return server.buildPrepackBundle(bundleOptions)
+    .then(function(p) {
+      server.end();
+      return p;
+    });
+};
+
 exports.buildPackageFromUrl =
 exports.buildBundleFromUrl = function(options, reqUrl) {
   var server = createNonPersistentServer(options);

packager/react-packager/src/Bundler/PrepackBundle.js

@@ -0,0 +1,149 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const fs = require('fs');
+
+class PrepackBundle {
+  constructor(sourceMapUrl) {
+    this._finalized = false;
+    this._moduleIds = Object.create(null);
+    this._modules = Object.create(null);
+    this._eagerModules = [];
+    this._mainModule = null;
+    this._assets = [];
+    this._sourceMapUrl = sourceMapUrl;
+  }
+
+  addModule(id, module, deps, isPolyfill) {
+    this._modules[module.sourcePath] = { module, deps };
+    this._moduleIds[id] = module.sourcePath;
+    if (isPolyfill) {
+      this._eagerModules.push(id);
+    }
+  }
+
+  addAsset(asset) {
+    this._assets.push(asset);
+  }
+
+  // Synchronously load a file path.
+  _loadFilename(path) {
+    const module = this._modules[path];
+    if (!module) {
+      throw new Error('Could not find file "' + path + '" in preloaded files.');
+    }
+    return module.module.code;
+  }
+
+  // Synchronously resolve a relative require from a parent module.
+  _resolveFilename(parentPath, relativePath) {
+    if (!parentPath) {
+      const resolvedPath = this._moduleIds[relativePath];
+      if (!resolvedPath) {
+        throw new Error('Could not resolve "' + relativePath + '".');
+      }
+      return resolvedPath;
+    }
+    const deps = this._modules[parentPath].deps;
+    const resolvedPath = deps[relativePath];
+    if (!resolvedPath) {
+      throw new Error(
+        'Could not resolve "' + relativePath + '" from "' + parentPath + '".'
+      );
+    }
+    return resolvedPath;
+  }
+
+  build(options) {
+    var prepack = require('prepack');
+
+    var batchedBridgeConfig = (options && options.batchedBridgeConfig) || null;
+    if (typeof batchedBridgeConfig === 'string') {
+      batchedBridgeConfig = JSON.parse(
+        fs.readFileSync(batchedBridgeConfig, 'utf-8')
+      );
+    }
+
+    var options = {
+      batchedBridgeConfig: batchedBridgeConfig,
+      environment: 'react-native',
+      resolveFilename: this._resolveFilename.bind(this),
+      loadFilename: this._loadFilename.bind(this),
+      eagerModules: this._eagerModules
+    };
+
+    return prepack.compileModule(this._mainModule, options);
+  }
+
+  finalize(options) {
+    options = options || {};
+    if (options.runMainModule) {
+      options.runBeforeMainModule.forEach(this._addRequireCall, this);
+      this._mainModule = options.mainModuleId;
+    }
+
+    Object.freeze(this._moduleIds);
+    Object.freeze(this._modules);
+    Object.freeze(this._assets);
+    Object.freeze(this._eagerModules);
+    this._finalized = true;
+  }
+
+  _addRequireCall(moduleId) {
+    this._eagerModules.push(moduleId);
+  }
+
+  _assertFinalized() {
+    if (!this._finalized) {
+      throw new Error('Bundle needs to be finalized before getting any source');
+    }
+  }
+
+  getAssets() {
+    return this._assets;
+  }
+
+  toJSON() {
+    if (!this._finalized) {
+      throw new Error('Cannot serialize bundle unless finalized');
+    }
+
+    return {
+      modules: this._modules,
+      moduleIds: this._moduleIds,
+      assets: this._assets,
+      sourceMapUrl: this._sourceMapUrl,
+      mainModule: this._mainModule,
+      eagerModules: this._eagerModules,
+    };
+  }
+
+  static fromJSON(json) {
+    const bundle = new PrepackBundle(json.sourceMapUrl);
+    bundle._assets = json.assets;
+    bundle._moduleIds = json.moduleIds;
+    bundle._modules = json.modules;
+    bundle._sourceMapUrl = json.sourceMapUrl;
+
+    bundle._eagerModules = json.eagerModules;
+    bundle._mainModule = json.mainModule;
+
+    Object.freeze(bundle._moduleIds);
+    Object.freeze(bundle._modules);
+    Object.freeze(bundle._assets);
+    Object.freeze(bundle._eagerModules);
+
+    bundle._finalized = true;
+
+    return bundle;
+  }
+}
+
+module.exports = PrepackBundle;

packager/react-packager/src/Bundler/index.js

@@ -18,6 +18,7 @@ const Cache = require('../Cache');
 const Transformer = require('../JSTransformer');
 const Resolver = require('../Resolver');
 const Bundle = require('./Bundle');
+const PrepackBundle = require('./PrepackBundle');
 const Activity = require('../Activity');
 const ModuleTransport = require('../lib/ModuleTransport');
 const declareOpts = require('../lib/declareOpts');
@@ -171,7 +172,7 @@ class Bundler {
             if (bar) {
               bar.tick();
             }
-            return transformed;
+            return this._wrapTransformedModule(response, module, transformed);
           })
         )
       );
@@ -187,6 +188,68 @@ class Bundler {
     });
   }
 
+  prepackBundle({
+    entryFile,
+    runModule: runMainModule,
+    runBeforeMainModule,
+    sourceMapUrl,
+    dev: isDev,
+    platform,
+  }) {
+    const bundle = new PrepackBundle(sourceMapUrl);
+    const findEventId = Activity.startEvent('find dependencies');
+    let transformEventId;
+    let mainModuleId;
+
+    return this.getDependencies(entryFile, isDev, platform).then((response) => {
+      Activity.endEvent(findEventId);
+      transformEventId = Activity.startEvent('transform');
+
+      let bar;
+      if (process.stdout.isTTY) {
+        bar = new ProgressBar('transforming [:bar] :percent :current/:total', {
+          complete: '=',
+          incomplete: ' ',
+          width: 40,
+          total: response.dependencies.length,
+        });
+      }
+
+      mainModuleId = response.mainModuleId;
+
+      return Promise.all(
+        response.dependencies.map(
+          module => this._transformModule(
+            bundle,
+            response,
+            module,
+            platform
+          ).then(transformed => {
+            if (bar) {
+              bar.tick();
+            }
+
+            var deps = Object.create(null);
+            var pairs = response.getResolvedDependencyPairs(module);
+            if (pairs) {
+              pairs.forEach(pair => {
+                deps[pair[0]] = pair[1].path;
+              });
+            }
+
+            return module.getName().then(name => {
+              bundle.addModule(name, transformed, deps, module.isPolyfill());
+            });
+          })
+        )
+      );
+    }).then(() => {
+      Activity.endEvent(transformEventId);
+      bundle.finalize({runBeforeMainModule, runMainModule, mainModuleId });
+      return bundle;
+    });
+  }
+
   invalidateFile(filePath) {
     this._transformer.invalidateFile(filePath);
   }
@@ -228,35 +291,32 @@ class Bundler {
   }
 
   _transformModule(bundle, response, module, platform = null) {
-    let transform;
-
     if (module.isAsset_DEPRECATED()) {
-      transform = this.generateAssetModule_DEPRECATED(bundle, module);
+      return this.generateAssetModule_DEPRECATED(bundle, module);
     } else if (module.isAsset()) {
-      transform = this.generateAssetModule(bundle, module, platform);
+      return this.generateAssetModule(bundle, module, platform);
     } else if (module.isJSON()) {
-      transform = generateJSONModule(module);
+      return generateJSONModule(module);
     } else {
-      transform = this._transformer.loadFileAndTransform(
+      return this._transformer.loadFileAndTransform(
         path.resolve(module.path)
       );
     }
+  }
 
-    const resolver = this._resolver;
-    return transform.then(
-      transformed => resolver.wrapModule(
-        response,
-        module,
-        transformed.code
-      ).then(
-        code => new ModuleTransport({
-          code: code,
-          map: transformed.map,
-          sourceCode: transformed.sourceCode,
-          sourcePath: transformed.sourcePath,
-          virtual: transformed.virtual,
-        })
-      )
+  _wrapTransformedModule(response, module, transformed) {
+    return this._resolver.wrapModule(
+      response,
+      module,
+      transformed.code
+    ).then(
+      code => new ModuleTransport({
+        code: code,
+        map: transformed.map,
+        sourceCode: transformed.sourceCode,
+        sourcePath: transformed.sourcePath,
+        virtual: transformed.virtual,
+      })
     );
   }
 

packager/react-packager/src/Server/index.js

@@ -190,6 +190,17 @@ class Server {
     });
   }
 
+  buildPrepackBundle(options) {
+    return Promise.resolve().then(() => {
+      if (!options.platform) {
+        options.platform = getPlatformExtension(options.entryFile);
+      }
+
+      const opts = bundleOpts(options);
+      return this._bundler.prepackBundle(opts);
+    });
+  }
+
   buildBundleFromUrl(reqUrl) {
     const options = this._getOptionsFromUrl(reqUrl);
     return this.buildBundle(options);

packager/react-packager/src/SocketInterface/SocketClient.js

@@ -9,6 +9,7 @@
 'use strict';
 
 const Bundle = require('../Bundler/Bundle');
+const PrepackBundle = require('../Bundler/PrepackBundle');
 const Promise = require('promise');
 const bser = require('bser');
 const debug = require('debug')('ReactNativePackager:SocketClient');
@@ -100,6 +101,13 @@ class SocketClient {
     }).then(json => Bundle.fromJSON(json));
   }
 
+  buildPrepackBundle(options) {
+    return this._send({
+      type: 'buildPrepackBundle',
+      data: options,
+    }).then(json => PrepackBundle.fromJSON(json));
+  }
+
   _send(message) {
     message.id = uid();
     this._sock.write(bser.dumpToBuffer(message));

packager/react-packager/src/SocketInterface/SocketServer.js

@@ -121,6 +121,14 @@ class SocketServer {
         );
         break;
 
+      case 'buildPrepackBundle':
+        this._jobs++;
+        this._packagerServer.buildPrepackBundle(m.data).then(
+          (result) => this._reply(sock, m.id, 'result', result),
+          handleError,
+        );
+        break;
+
       case 'getOrderedDependencyPaths':
         this._jobs++;
         this._packagerServer.getOrderedDependencyPaths(m.data).then(