From 43f18ffd087280e6df1e4900deb6fe201b87741a Mon Sep 17 00:00:00 2001 From: Sebastian Markbage Date: Fri, 20 Nov 2015 20:17:17 -0800 Subject: [PATCH] Add infra for Prepack build option Summary: This adds a build option for using Prepack (an experimental packager) to build a bundle. It doesn't actually take on the npm package dependency because it's not published/open source (yet). This will be used while we experiment and should be maintained as the build system changes so that we can continue getting fresh builds. I found that saveBundleAndMap and processBundle were over abstracted and got in my way so I inlined it and removed the unit tests because the unit test was testing trivial code that is likely to change interface. I went with a separate build phase and a separate Bundle class even though there are a lot of commonalities. I imagine that the requirements for Prepack will continue to diverge. Especially for source maps but a larger refactor could try to unify these a bit more. The fact that modules are wrapped before the write phase seems to be an unfortunate architecture that makes this difficult. Closes https://github.com/facebook/react-native/pull/4226 Reviewed By: amasad Differential Revision: D2673760 Pulled By: sebmarkbage fb-gh-sync-id: 299ccc42e4be1d9dee19ade443ea3388db2e39a8 --- .../bundle/__tests__/saveBundleAndMap-test.js | 63 -------- local-cli/bundle/buildBundle.js | 108 +++++++++++-- local-cli/bundle/bundleCommandLineArgs.js | 8 + local-cli/bundle/processBundle.js | 29 ---- .../{saveBundleAndMap.js => saveAssets.js} | 22 +-- packager/react-packager/index.js | 9 ++ .../src/Bundler/PrepackBundle.js | 149 ++++++++++++++++++ packager/react-packager/src/Bundler/index.js | 104 +++++++++--- packager/react-packager/src/Server/index.js | 11 ++ .../src/SocketInterface/SocketClient.js | 8 + .../src/SocketInterface/SocketServer.js | 8 + 11 files changed, 372 insertions(+), 147 deletions(-) delete mode 100644 local-cli/bundle/__tests__/saveBundleAndMap-test.js delete mode 100644 local-cli/bundle/processBundle.js rename local-cli/bundle/{saveBundleAndMap.js => saveAssets.js} (80%) create mode 100644 packager/react-packager/src/Bundler/PrepackBundle.js diff --git a/local-cli/bundle/__tests__/saveBundleAndMap-test.js b/local-cli/bundle/__tests__/saveBundleAndMap-test.js deleted file mode 100644 index 4bd516a05..000000000 --- a/local-cli/bundle/__tests__/saveBundleAndMap-test.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright (c) 2015-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ -'use strict'; - -jest.autoMockOff(); - -jest.mock('fs'); -jest.mock('../sign'); - -const saveBundleAndMap = require('../saveBundleAndMap'); -const fs = require('fs'); -const temp = require('temp'); - -const code = 'const foo = "bar";'; -const map = JSON.stringify({ - version: 3, - file: 'foo.js.map', - sources: ['foo.js'], - sourceRoot: '/', - names: ['bar'], - mappings: 'AAA0B,kBAAhBA,QAAOC,SACjBD,OAAOC,OAAO' -}); - -describe('saveBundleAndMap', () => { - beforeEach(() => { - fs.writeFileSync = jest.genMockFn(); - }); - - it('should save bundle', () => { - const codeWithMap = {code: code}; - const bundleOutput = temp.path({suffix: '.bundle'}); - - saveBundleAndMap( - codeWithMap, - 'ios', - bundleOutput, - 'utf8', - ); - - expect(fs.writeFileSync.mock.calls[0]).toEqual([bundleOutput, code, 'utf8']); - }); - - it('should save sourcemaps if required so', () => { - const codeWithMap = {code: code, map: map}; - const bundleOutput = temp.path({suffix: '.bundle'}); - const sourceMapOutput = temp.path({suffix: '.map'}); - saveBundleAndMap( - codeWithMap, - 'ios', - bundleOutput, - 'utf8', - sourceMapOutput - ); - - expect(fs.writeFileSync.mock.calls[1]).toEqual([sourceMapOutput, map]); - }); -}); diff --git a/local-cli/bundle/buildBundle.js b/local-cli/bundle/buildBundle.js index f69eea5a8..5cb752a43 100644 --- a/local-cli/bundle/buildBundle.js +++ b/local-cli/bundle/buildBundle.js @@ -8,11 +8,57 @@ */ 'use strict'; +const fs = require('fs'); const log = require('../util/log').out('bundle'); -const processBundle = require('./processBundle'); const Promise = require('promise'); const ReactPackager = require('../../packager/react-packager'); -const saveBundleAndMap = require('./saveBundleAndMap'); +const saveAssets = require('./saveAssets'); + +const sign = require('./sign'); + +function saveBundleAndMap( + bundle, + bundleOutput, + encoding, + sourcemapOutput, + dev +) { + log('start'); + let codeWithMap; + if (!dev) { + codeWithMap = bundle.getMinifiedSourceAndMap(dev); + } else { + codeWithMap = { + code: bundle.getSource({ dev }), + map: JSON.stringify(bundle.getSourceMap({ dev })), + }; + } + log('finish'); + + log('Writing bundle output to:', bundleOutput); + fs.writeFileSync(bundleOutput, sign(codeWithMap.code), encoding); + log('Done writing bundle output'); + + if (sourcemapOutput) { + log('Writing sourcemap output to:', sourcemapOutput); + fs.writeFileSync(sourcemapOutput, codeWithMap.map); + log('Done writing sourcemap output'); + } +} + +function savePrepackBundleAndMap( + bundle, + bundleOutput, + sourcemapOutput, + bridgeConfig +) { + log('Writing prepack bundle output to:', bundleOutput); + const result = bundle.build({ + batchedBridgeConfig: bridgeConfig + }); + fs.writeFileSync(bundleOutput, result, 'ucs-2'); + log('Done writing prepack bundle output'); +} function buildBundle(args, config) { return new Promise((resolve, reject) => { @@ -36,24 +82,56 @@ function buildBundle(args, config) { platform: args.platform, }; - resolve(ReactPackager.createClientFor(options).then(client => { - log('Created ReactPackager'); - return client.buildBundle(requestOpts) + const prepack = args.prepack; + + const client = ReactPackager.createClientFor(options); + + client.then(() => log('Created ReactPackager')); + + // Build and save the bundle + let bundle; + if (prepack) { + bundle = client.then(c => c.buildPrepackBundle(requestOpts)) .then(outputBundle => { - log('Closing client'); - client.close(); + savePrepackBundleAndMap( + outputBundle, + args['bundle-output'], + args['sourcemap-output'], + args['bridge-config'] + ); return outputBundle; - }) - .then(outputBundle => processBundle(outputBundle, args.dev)) - .then(outputBundle => saveBundleAndMap( - outputBundle, + }); + } else { + bundle = client.then(c => c.buildBundle(requestOpts)) + .then(outputBundle => { + saveBundleAndMap( + outputBundle, + args['bundle-output'], + args['bundle-encoding'], + args['sourcemap-output'], + args.dev + ); + return outputBundle; + }); + } + + // When we're done bundling, close the client + bundle.then(() => client.then(c => { + log('Closing client'); + c.close(); + })); + + // Save the assets of the bundle + const assets = bundle + .then(outputBundle => outputBundle.getAssets()) + .then(outputAssets => saveAssets( + outputAssets, args.platform, - args['bundle-output'], - args['bundle-encoding'], - args['sourcemap-output'], args['assets-dest'] )); - })); + + // When we're done saving the assets, we're done. + resolve(assets); }); } diff --git a/local-cli/bundle/bundleCommandLineArgs.js b/local-cli/bundle/bundleCommandLineArgs.js index 353c7ea5f..2597a7f13 100644 --- a/local-cli/bundle/bundleCommandLineArgs.js +++ b/local-cli/bundle/bundleCommandLineArgs.js @@ -27,6 +27,14 @@ module.exports = [ command: 'dev', description: 'If false, warnings are disabled and the bundle is minified', default: true, + }, { + command: 'prepack', + description: 'If true, the output bundle will use the Prepack format.', + default: false + }, { + command: 'bridge-config', + description: 'File name of a a JSON export of __fbBatchedBridgeConfig. Used by Prepack. Ex. ./bridgeconfig.json', + type: 'string' }, { command: 'bundle-output', description: 'File name where to store the resulting bundle, ex. /tmp/groups.bundle', diff --git a/local-cli/bundle/processBundle.js b/local-cli/bundle/processBundle.js deleted file mode 100644 index 3a86a61f5..000000000 --- a/local-cli/bundle/processBundle.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright (c) 2015-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ -'use strict'; - -const log = require('../util/log').out('bundle'); - -function processBundle(input, dev) { - log('start'); - let bundle; - if (!dev) { - bundle = input.getMinifiedSourceAndMap(dev); - } else { - bundle = { - code: input.getSource({ dev }), - map: JSON.stringify(input.getSourceMap({ dev })), - }; - } - bundle.assets = input.getAssets(); - log('finish'); - return bundle; -} - -module.exports = processBundle; diff --git a/local-cli/bundle/saveBundleAndMap.js b/local-cli/bundle/saveAssets.js similarity index 80% rename from local-cli/bundle/saveBundleAndMap.js rename to local-cli/bundle/saveAssets.js index bb6bf2291..1dd88512b 100644 --- a/local-cli/bundle/saveBundleAndMap.js +++ b/local-cli/bundle/saveAssets.js @@ -14,26 +14,12 @@ const getAssetDestPathAndroid = require('./getAssetDestPathAndroid'); const getAssetDestPathIOS = require('./getAssetDestPathIOS'); const log = require('../util/log').out('bundle'); const path = require('path'); -const sign = require('./sign'); -function saveBundleAndMap( - codeWithMap, +function saveAssets( + assets, platform, - bundleOutput, - encoding, - sourcemapOutput, assetsDest ) { - log('Writing bundle output to:', bundleOutput); - fs.writeFileSync(bundleOutput, sign(codeWithMap.code), encoding); - log('Done writing bundle output'); - - if (sourcemapOutput) { - log('Writing sourcemap output to:', sourcemapOutput); - fs.writeFileSync(sourcemapOutput, codeWithMap.map); - log('Done writing sourcemap output'); - } - if (!assetsDest) { console.warn('Assets destination folder is not set, skipping...'); return Promise.resolve(); @@ -44,7 +30,7 @@ function saveBundleAndMap( : getAssetDestPathIOS; const filesToCopy = Object.create(null); // Map src -> dest - codeWithMap.assets + assets .filter(asset => !asset.deprecated) .forEach(asset => asset.scales.forEach((scale, idx) => { @@ -94,4 +80,4 @@ function copy(src, dest, callback) { }); } -module.exports = saveBundleAndMap; +module.exports = saveAssets; diff --git a/packager/react-packager/index.js b/packager/react-packager/index.js index e0a29684d..e1a294f40 100644 --- a/packager/react-packager/index.js +++ b/packager/react-packager/index.js @@ -35,6 +35,15 @@ exports.buildBundle = function(options, bundleOptions) { }); }; +exports.buildPrepackBundle = function(options, bundleOptions) { + var server = createNonPersistentServer(options); + return server.buildPrepackBundle(bundleOptions) + .then(function(p) { + server.end(); + return p; + }); +}; + exports.buildPackageFromUrl = exports.buildBundleFromUrl = function(options, reqUrl) { var server = createNonPersistentServer(options); diff --git a/packager/react-packager/src/Bundler/PrepackBundle.js b/packager/react-packager/src/Bundler/PrepackBundle.js new file mode 100644 index 000000000..123d3297a --- /dev/null +++ b/packager/react-packager/src/Bundler/PrepackBundle.js @@ -0,0 +1,149 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +const fs = require('fs'); + +class PrepackBundle { + constructor(sourceMapUrl) { + this._finalized = false; + this._moduleIds = Object.create(null); + this._modules = Object.create(null); + this._eagerModules = []; + this._mainModule = null; + this._assets = []; + this._sourceMapUrl = sourceMapUrl; + } + + addModule(id, module, deps, isPolyfill) { + this._modules[module.sourcePath] = { module, deps }; + this._moduleIds[id] = module.sourcePath; + if (isPolyfill) { + this._eagerModules.push(id); + } + } + + addAsset(asset) { + this._assets.push(asset); + } + + // Synchronously load a file path. + _loadFilename(path) { + const module = this._modules[path]; + if (!module) { + throw new Error('Could not find file "' + path + '" in preloaded files.'); + } + return module.module.code; + } + + // Synchronously resolve a relative require from a parent module. + _resolveFilename(parentPath, relativePath) { + if (!parentPath) { + const resolvedPath = this._moduleIds[relativePath]; + if (!resolvedPath) { + throw new Error('Could not resolve "' + relativePath + '".'); + } + return resolvedPath; + } + const deps = this._modules[parentPath].deps; + const resolvedPath = deps[relativePath]; + if (!resolvedPath) { + throw new Error( + 'Could not resolve "' + relativePath + '" from "' + parentPath + '".' + ); + } + return resolvedPath; + } + + build(options) { + var prepack = require('prepack'); + + var batchedBridgeConfig = (options && options.batchedBridgeConfig) || null; + if (typeof batchedBridgeConfig === 'string') { + batchedBridgeConfig = JSON.parse( + fs.readFileSync(batchedBridgeConfig, 'utf-8') + ); + } + + var options = { + batchedBridgeConfig: batchedBridgeConfig, + environment: 'react-native', + resolveFilename: this._resolveFilename.bind(this), + loadFilename: this._loadFilename.bind(this), + eagerModules: this._eagerModules + }; + + return prepack.compileModule(this._mainModule, options); + } + + finalize(options) { + options = options || {}; + if (options.runMainModule) { + options.runBeforeMainModule.forEach(this._addRequireCall, this); + this._mainModule = options.mainModuleId; + } + + Object.freeze(this._moduleIds); + Object.freeze(this._modules); + Object.freeze(this._assets); + Object.freeze(this._eagerModules); + this._finalized = true; + } + + _addRequireCall(moduleId) { + this._eagerModules.push(moduleId); + } + + _assertFinalized() { + if (!this._finalized) { + throw new Error('Bundle needs to be finalized before getting any source'); + } + } + + getAssets() { + return this._assets; + } + + toJSON() { + if (!this._finalized) { + throw new Error('Cannot serialize bundle unless finalized'); + } + + return { + modules: this._modules, + moduleIds: this._moduleIds, + assets: this._assets, + sourceMapUrl: this._sourceMapUrl, + mainModule: this._mainModule, + eagerModules: this._eagerModules, + }; + } + + static fromJSON(json) { + const bundle = new PrepackBundle(json.sourceMapUrl); + bundle._assets = json.assets; + bundle._moduleIds = json.moduleIds; + bundle._modules = json.modules; + bundle._sourceMapUrl = json.sourceMapUrl; + + bundle._eagerModules = json.eagerModules; + bundle._mainModule = json.mainModule; + + Object.freeze(bundle._moduleIds); + Object.freeze(bundle._modules); + Object.freeze(bundle._assets); + Object.freeze(bundle._eagerModules); + + bundle._finalized = true; + + return bundle; + } +} + +module.exports = PrepackBundle; diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index 82a0282be..5f5d79590 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -18,6 +18,7 @@ const Cache = require('../Cache'); const Transformer = require('../JSTransformer'); const Resolver = require('../Resolver'); const Bundle = require('./Bundle'); +const PrepackBundle = require('./PrepackBundle'); const Activity = require('../Activity'); const ModuleTransport = require('../lib/ModuleTransport'); const declareOpts = require('../lib/declareOpts'); @@ -171,7 +172,7 @@ class Bundler { if (bar) { bar.tick(); } - return transformed; + return this._wrapTransformedModule(response, module, transformed); }) ) ); @@ -187,6 +188,68 @@ class Bundler { }); } + prepackBundle({ + entryFile, + runModule: runMainModule, + runBeforeMainModule, + sourceMapUrl, + dev: isDev, + platform, + }) { + const bundle = new PrepackBundle(sourceMapUrl); + const findEventId = Activity.startEvent('find dependencies'); + let transformEventId; + let mainModuleId; + + return this.getDependencies(entryFile, isDev, platform).then((response) => { + Activity.endEvent(findEventId); + transformEventId = Activity.startEvent('transform'); + + let bar; + if (process.stdout.isTTY) { + bar = new ProgressBar('transforming [:bar] :percent :current/:total', { + complete: '=', + incomplete: ' ', + width: 40, + total: response.dependencies.length, + }); + } + + mainModuleId = response.mainModuleId; + + return Promise.all( + response.dependencies.map( + module => this._transformModule( + bundle, + response, + module, + platform + ).then(transformed => { + if (bar) { + bar.tick(); + } + + var deps = Object.create(null); + var pairs = response.getResolvedDependencyPairs(module); + if (pairs) { + pairs.forEach(pair => { + deps[pair[0]] = pair[1].path; + }); + } + + return module.getName().then(name => { + bundle.addModule(name, transformed, deps, module.isPolyfill()); + }); + }) + ) + ); + }).then(() => { + Activity.endEvent(transformEventId); + bundle.finalize({runBeforeMainModule, runMainModule, mainModuleId }); + return bundle; + }); + } + invalidateFile(filePath) { this._transformer.invalidateFile(filePath); } @@ -228,35 +291,32 @@ class Bundler { } _transformModule(bundle, response, module, platform = null) { - let transform; - if (module.isAsset_DEPRECATED()) { - transform = this.generateAssetModule_DEPRECATED(bundle, module); + return this.generateAssetModule_DEPRECATED(bundle, module); } else if (module.isAsset()) { - transform = this.generateAssetModule(bundle, module, platform); + return this.generateAssetModule(bundle, module, platform); } else if (module.isJSON()) { - transform = generateJSONModule(module); + return generateJSONModule(module); } else { - transform = this._transformer.loadFileAndTransform( + return this._transformer.loadFileAndTransform( path.resolve(module.path) ); } + } - const resolver = this._resolver; - return transform.then( - transformed => resolver.wrapModule( - response, - module, - transformed.code - ).then( - code => new ModuleTransport({ - code: code, - map: transformed.map, - sourceCode: transformed.sourceCode, - sourcePath: transformed.sourcePath, - virtual: transformed.virtual, - }) - ) + _wrapTransformedModule(response, module, transformed) { + return this._resolver.wrapModule( + response, + module, + transformed.code + ).then( + code => new ModuleTransport({ + code: code, + map: transformed.map, + sourceCode: transformed.sourceCode, + sourcePath: transformed.sourcePath, + virtual: transformed.virtual, + }) ); } diff --git a/packager/react-packager/src/Server/index.js b/packager/react-packager/src/Server/index.js index 1abb78cf6..28c5b4d1b 100644 --- a/packager/react-packager/src/Server/index.js +++ b/packager/react-packager/src/Server/index.js @@ -190,6 +190,17 @@ class Server { }); } + buildPrepackBundle(options) { + return Promise.resolve().then(() => { + if (!options.platform) { + options.platform = getPlatformExtension(options.entryFile); + } + + const opts = bundleOpts(options); + return this._bundler.prepackBundle(opts); + }); + } + buildBundleFromUrl(reqUrl) { const options = this._getOptionsFromUrl(reqUrl); return this.buildBundle(options); diff --git a/packager/react-packager/src/SocketInterface/SocketClient.js b/packager/react-packager/src/SocketInterface/SocketClient.js index 41ce0df14..a344f5d97 100644 --- a/packager/react-packager/src/SocketInterface/SocketClient.js +++ b/packager/react-packager/src/SocketInterface/SocketClient.js @@ -9,6 +9,7 @@ 'use strict'; const Bundle = require('../Bundler/Bundle'); +const PrepackBundle = require('../Bundler/PrepackBundle'); const Promise = require('promise'); const bser = require('bser'); const debug = require('debug')('ReactNativePackager:SocketClient'); @@ -100,6 +101,13 @@ class SocketClient { }).then(json => Bundle.fromJSON(json)); } + buildPrepackBundle(options) { + return this._send({ + type: 'buildPrepackBundle', + data: options, + }).then(json => PrepackBundle.fromJSON(json)); + } + _send(message) { message.id = uid(); this._sock.write(bser.dumpToBuffer(message)); diff --git a/packager/react-packager/src/SocketInterface/SocketServer.js b/packager/react-packager/src/SocketInterface/SocketServer.js index 8258873a4..56562e273 100644 --- a/packager/react-packager/src/SocketInterface/SocketServer.js +++ b/packager/react-packager/src/SocketInterface/SocketServer.js @@ -121,6 +121,14 @@ class SocketServer { ); break; + case 'buildPrepackBundle': + this._jobs++; + this._packagerServer.buildPrepackBundle(m.data).then( + (result) => this._reply(sock, m.id, 'result', result), + handleError, + ); + break; + case 'getOrderedDependencyPaths': this._jobs++; this._packagerServer.getOrderedDependencyPaths(m.data).then(