From f99468de6500d516bc461cd26bdb148d4758eeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Bigio?= Date: Sun, 13 Mar 2016 11:13:40 -0700 Subject: [PATCH] Sourcemaps support for RAM Summary:This rev adds support for production sourcemaps on RAM. When we inject a module into JSC we use the original `sourceURL` and specify the `startingLineNumber` of the module relative to a "regular" bundle. By doing so, when an error is thrown, JSC will include the provided `sourceURL` as the filename and will use the indicated `startingLineNumber` to figure out on which line the error actually occurred. To make things a bit simpler and avoid having to deal with columns, we tweak the generated bundle so that each module starts on a new line. Since we cannot assure that each module's code will be on a single line as the minifier might break it on multiple (UglifyJS does so due to a bug on old versions of Chrome), we include on the index the line number that should be used when invoking `JSEvaluateScript`. Since the module length was not being used we replaced the placeholder we have there for the line number. Reviewed By: javache Differential Revision: D2997520 fb-gh-sync-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f shipit-source-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f --- React/Executors/RCTJSCExecutor.m | 43 +++++++++----- .../bundle/output/unbundle/as-indexed-file.js | 57 ++++++++++++------ .../output/unbundle/buildUnbundleSourcemap.js | 59 +++++++++++++++++++ .../bundle/output/unbundle/write-sourcemap.js | 2 +- packager/react-packager/src/Bundler/Bundle.js | 19 +++--- .../src/Bundler/__tests__/Bundle-test.js | 15 +++-- .../src/Bundler/__tests__/Bundler-test.js | 1 + packager/react-packager/src/Bundler/index.js | 26 ++++++-- packager/react-packager/src/Resolver/index.js | 4 ++ .../react-packager/src/lib/ModuleTransport.js | 1 + 10 files changed, 174 insertions(+), 53 deletions(-) create mode 100644 local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js diff --git a/React/Executors/RCTJSCExecutor.m b/React/Executors/RCTJSCExecutor.m index af94cf2e1..de36fe72c 100644 --- a/React/Executors/RCTJSCExecutor.m +++ b/React/Executors/RCTJSCExecutor.m @@ -32,10 +32,10 @@ NSString *const RCTJavaScriptContextCreatedNotification = @"RCTJavaScriptContext static NSString *const RCTJSCProfilerEnabledDefaultsKey = @"RCTJSCProfilerEnabled"; -// TODO: add lineNo typedef struct ModuleData { uint32_t offset; + uint32_t lineNo; } ModuleData; @interface RCTJavaScriptContext : NSObject @@ -668,6 +668,14 @@ static void freeModuleData(__unused CFAllocatorRef allocator, void *ptr) free(ptr); } +static uint32_t readUint32(const void **ptr) { + uint32_t data; + memcpy(&data, *ptr, sizeof(uint32_t)); + data = NSSwapLittleIntToHost(data); + *ptr += sizeof(uint32_t); + return data; +} + - (NSData *)loadRAMBundle:(NSData *)script { __weak RCTJSCExecutor *weakSelf = self; @@ -679,8 +687,9 @@ static void freeModuleData(__unused CFAllocatorRef allocator, void *ptr) ModuleData *moduleData = (ModuleData *)CFDictionaryGetValue(strongSelf->_jsModules, moduleName.UTF8String); JSStringRef module = JSStringCreateWithUTF8CString((const char *)_bundle.bytes + moduleData->offset); + int lineNo = [moduleName isEqual:@""] ? 0 : moduleData->lineNo; JSValueRef jsError = NULL; - JSValueRef result = JSEvaluateScript(strongSelf->_context.ctx, module, NULL, strongSelf->_bundleURL, NULL, &jsError); + JSValueRef result = JSEvaluateScript(strongSelf->_context.ctx, module, NULL, strongSelf->_bundleURL, lineNo, NULL); CFDictionaryRemoveValue(strongSelf->_jsModules, moduleName.UTF8String); JSStringRelease(module); @@ -706,26 +715,28 @@ static void freeModuleData(__unused CFAllocatorRef allocator, void *ptr) memcpy(&tableLength, bytes + currentOffset, sizeof(tableLength)); tableLength = NSSwapLittleIntToHost(tableLength); - uint32_t baseOffset = currentOffset + tableLength; + // offset where the code starts on the bundle + const uint32_t baseOffset = currentOffset + tableLength; + + // skip table length currentOffset += sizeof(baseOffset); - while (currentOffset < baseOffset) { - const char *moduleName = (const char *)bytes + currentOffset; - uint32_t offset; + // pointer to first byte out of the index + const uint8_t *endOfTable = bytes + baseOffset; + + // pointer to current position on table + const uint8_t *tablePos = bytes + 2 * sizeof(uint32_t); // skip magic number and table length + + while (tablePos < endOfTable) { + const char *moduleName = (const char *)tablePos; // the space allocated for each module's metada gets freed when the module is injected into JSC on `nativeRequire` - ModuleData *moduleData = malloc(sizeof(moduleData)); + ModuleData *moduleData = malloc(sizeof(ModuleData)); - // skip module name and null byte terminator - currentOffset += strlen(moduleName) + 1; + tablePos += strlen(moduleName) + 1; // null byte terminator - // read and save offset - memcpy(&offset, bytes + currentOffset, sizeof(offset)); - offset = NSSwapLittleIntToHost(offset); - moduleData->offset = baseOffset + offset; - - // TODO: replace length with lineNo - currentOffset += sizeof(offset) * 2; // skip both offset and lenght + moduleData->offset = baseOffset + readUint32((const void **)&tablePos); + moduleData->lineNo = readUint32((const void **)&tablePos); CFDictionarySetValue(_jsModules, moduleName, moduleData); } diff --git a/local-cli/bundle/output/unbundle/as-indexed-file.js b/local-cli/bundle/output/unbundle/as-indexed-file.js index f09b1f10d..870ffdb1f 100644 --- a/local-cli/bundle/output/unbundle/as-indexed-file.js +++ b/local-cli/bundle/output/unbundle/as-indexed-file.js @@ -8,6 +8,7 @@ */ 'use strict'; +const buildUnbundleSourcemap = require('./buildUnbundleSourcemap'); const fs = require('fs'); const Promise = require('promise'); const writeSourceMap = require('./write-sourcemap'); @@ -26,23 +27,27 @@ function saveAsIndexedFile(bundle, options, log) { const { 'bundle-output': bundleOutput, 'bundle-encoding': encoding, - dev, 'sourcemap-output': sourcemapOutput, } = options; log('start'); - const {startupCode, modules} = bundle.getUnbundle({minify: !dev}); + const {startupCode, modules} = bundle.getUnbundle(); log('finish'); log('Writing unbundle output to:', bundleOutput); const writeUnbundle = writeBuffers( fs.createWriteStream(bundleOutput), buildTableAndContents(startupCode, modules, encoding) - ); + ).then(() => log('Done writing unbundle output')); - writeUnbundle.then(() => log('Done writing unbundle output')); - - return Promise.all([writeUnbundle, writeSourceMap(sourcemapOutput, '', log)]); + return Promise.all([ + writeUnbundle, + writeSourceMap( + sourcemapOutput, + buildUnbundleSourcemap(bundle), + log, + ), + ]); } /* global Buffer: true */ @@ -60,13 +65,16 @@ function writeBuffers(stream, buffers) { }); } -const moduleToBuffer = ({name, code}, encoding) => ({ - name, - buffer: Buffer.concat([ - Buffer(code, encoding), - nullByteBuffer // create \0-terminated strings - ]) -}); +function moduleToBuffer(name, code, encoding) { + return { + name, + linesCount: code.split('\n').length, + buffer: Buffer.concat([ + Buffer(code, encoding), + nullByteBuffer // create \0-terminated strings + ]) + }; +} function uInt32Buffer(n) { const buffer = Buffer(4); @@ -82,23 +90,28 @@ function buildModuleTable(buffers) { // entry: // - module_id: NUL terminated utf8 string // - module_offset: uint_32 offset into the module string - // - module_length: uint_32 length of the module string, including terminating NUL byte + // - module_line: uint_32 line on which module starts on the bundle const numBuffers = buffers.length; const tableLengthBuffer = uInt32Buffer(0); let tableLength = 4; // the table length itself, 4 == tableLengthBuffer.length let currentOffset = 0; + let currentLine = 1; const offsetTable = [tableLengthBuffer]; for (let i = 0; i < numBuffers; i++) { - const {name, buffer: {length}} = buffers[i]; + const {name, linesCount, buffer: {length}} = buffers[i]; + const entry = Buffer.concat([ Buffer(i === 0 ? MAGIC_STARTUP_MODULE_ID : name, 'utf8'), nullByteBuffer, uInt32Buffer(currentOffset), - uInt32Buffer(length) + uInt32Buffer(currentLine), ]); + + currentLine += linesCount - 1; + currentOffset += length; tableLength += entry.length; offsetTable.push(entry); @@ -110,14 +123,22 @@ function buildModuleTable(buffers) { function buildModuleBuffers(startupCode, modules, encoding) { return ( - [moduleToBuffer({name: '', code: startupCode}, encoding)] - .concat(modules.map(module => moduleToBuffer(module, encoding))) + [moduleToBuffer('', startupCode, encoding, true)].concat( + modules.map(module => + moduleToBuffer( + module.name, + module.code + '\n', // each module starts on a newline + encoding, + ) + ) + ) ); } function buildTableAndContents(startupCode, modules, encoding) { const buffers = buildModuleBuffers(startupCode, modules, encoding); const table = buildModuleTable(buffers, encoding); + return [fileHeader, table].concat(buffers.map(({buffer}) => buffer)); } diff --git a/local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js b/local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js new file mode 100644 index 000000000..6f7b48d54 --- /dev/null +++ b/local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +const sourceMap = require('source-map'); +const SourceMapConsumer = sourceMap.SourceMapConsumer; + +/** + * Builds the sourcemaps for any type of unbundle provided the Bundle that + * contains the modules reachable from the entry point. + * + * The generated sourcemaps correspond to a regular bundle on which each module + * starts on a new line. Depending on the type of unbundle you're using, you + * will have to pipe the line number to native and use it when injecting the + * module's code into JSC. This way, we'll trick JSC to believe all the code is + * on a single big regular bundle where as it could be on an indexed bundle or + * as sparsed assets. + */ +function buildUnbundleSourcemap(bundle) { + const generator = new sourceMap.SourceMapGenerator({}); + const nonPolyfillModules = bundle.getModules().filter(module => + !module.polyfill + ); + + let offset = 1; + nonPolyfillModules.forEach(module => { + if (module.map) { // assets have no sourcemap + const consumer = new SourceMapConsumer(module.map); + consumer.eachMapping(mapping => { + generator.addMapping({ + original: { + line: mapping.originalLine, + column: mapping.originalColumn, + }, + generated: { + line: mapping.generatedLine + offset, + column: mapping.generatedColumn, + }, + source: module.sourcePath, + }); + }); + + generator.setSourceContent(module.sourcePath, module.sourceCode); + } + + // some modules span more than 1 line + offset += module.code.split('\n').length; + }); + + return generator.toString(); +} + +module.exports = buildUnbundleSourcemap; diff --git a/local-cli/bundle/output/unbundle/write-sourcemap.js b/local-cli/bundle/output/unbundle/write-sourcemap.js index b194b9922..d74a70260 100644 --- a/local-cli/bundle/output/unbundle/write-sourcemap.js +++ b/local-cli/bundle/output/unbundle/write-sourcemap.js @@ -16,7 +16,7 @@ function writeSourcemap(fileName, contents, log) { return Promise.resolve(); } log('Writing sourcemap output to:', fileName); - const writeMap = writeFile(fileName, '', null); + const writeMap = writeFile(fileName, contents, null); writeMap.then(() => log('Done writing sourcemap output')); return writeMap; } diff --git a/packager/react-packager/src/Bundler/Bundle.js b/packager/react-packager/src/Bundler/Bundle.js index 2a947f291..4771f678b 100644 --- a/packager/react-packager/src/Bundler/Bundle.js +++ b/packager/react-packager/src/Bundler/Bundle.js @@ -16,9 +16,6 @@ const crypto = require('crypto'); const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL='; -const getCode = x => x.code; -const getNameAndCode = ({name, code}) => ({name, code}); - class Bundle extends BundleBase { constructor({sourceMapUrl, minify} = {}) { super(); @@ -40,6 +37,7 @@ class Bundle extends BundleBase { map: moduleTransport.map, meta: moduleTransport.meta, minify: this._minify, + polyfill: module.isPolyfill(), }).then(({code, map}) => { // If we get a map from the transformer we'll switch to a mode // were we're combining the source maps as opposed to @@ -113,20 +111,21 @@ class Bundle extends BundleBase { const modules = allModules .splice(prependedModules, allModules.length - requireCalls - prependedModules); - const startupCode = allModules.map(getCode).join('\n'); + const startupCode = allModules.map(({code}) => code).join('\n'); return { startupCode, - modules: modules.map(getNameAndCode) + modules: modules.map(({name, code, polyfill}) => + ({name, code, polyfill}) + ), }; } /** - * I found a neat trick in the sourcemap spec that makes it easy - * to concat sourcemaps. The `sections` field allows us to combine - * the sourcemap easily by adding an offset. Tested on chrome. - * Seems like it's not yet in Firefox but that should be fine for - * now. + * Combine each of the sourcemaps multiple modules have into a single big + * one. This works well thanks to a neat trick defined on the sourcemap spec + * that makes use of of the `sections` field to combine sourcemaps by adding + * an offset. This is supported only by Chrome for now. */ _getCombinedSourceMaps(options) { const result = { diff --git a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js index 2235ba500..d6fa569fa 100644 --- a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js +++ b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js @@ -123,7 +123,7 @@ describe('Bundle', () => { }; const promise = Promise.all( - moduleTransports.map(m => bundle.addModule(resolver, null, null, m))) + moduleTransports.map(m => bundle.addModule(resolver, null, {isPolyfill: () => false}, m))) .then(() => { expect(bundle.getModules()) .toEqual(moduleTransports); @@ -375,12 +375,19 @@ function resolverFor(code, map) { }; } -function addModule({bundle, code, sourceCode, sourcePath, map, virtual}) { +function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) { return bundle.addModule( resolverFor(code, map), null, - null, - createModuleTransport({code, sourceCode, sourcePath, map, virtual}) + {isPolyfill: () => polyfill}, + createModuleTransport({ + code, + sourceCode, + sourcePath, + map, + virtual, + polyfill, + }), ); } diff --git a/packager/react-packager/src/Bundler/__tests__/Bundler-test.js b/packager/react-packager/src/Bundler/__tests__/Bundler-test.js index ac37db896..e1a67b2dd 100644 --- a/packager/react-packager/src/Bundler/__tests__/Bundler-test.js +++ b/packager/react-packager/src/Bundler/__tests__/Bundler-test.js @@ -204,6 +204,7 @@ describe('Bundler', function() { transform: { dev: true, hot: false, + generateSourceMaps: false, projectRoots, } }, diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index 3c1c426df..832968078 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -230,8 +230,9 @@ class Bundler { platform, moduleSystemDeps = [], hot, + unbundle, entryModuleOnly, - resolutionResponse + resolutionResponse, }) { const onResolutionResponse = response => { bundle.setMainModuleId(response.mainModuleId); @@ -265,6 +266,7 @@ class Bundler { platform, bundle, hot, + unbundle, resolutionResponse, onResolutionResponse, finalizeBundle, @@ -316,6 +318,7 @@ class Bundler { platform, bundle, hot, + unbundle, resolutionResponse, onResolutionResponse = noop, onModuleTransformed = noop, @@ -336,8 +339,15 @@ class Bundler { }; } - resolutionResponse = this.getDependencies( - {entryFile, dev, platform, hot, onProgess, minify}); + resolutionResponse = this.getDependencies({ + entryFile, + dev, + platform, + hot, + onProgess, + minify, + generateSourceMaps: unbundle, + }); } return Promise.resolve(resolutionResponse).then(response => { @@ -391,10 +401,18 @@ class Bundler { minify = !dev, hot = false, recursive = true, + generateSourceMaps = false, onProgess, }) { return this.getTransformOptions( - entryFile, {dev, platform, hot, projectRoots: this._projectRoots} + entryFile, + { + dev, + platform, + hot, + generateSourceMaps, + projectRoots: this._projectRoots, + }, ).then(transformSpecificOptions => { const transformOptions = { minify, diff --git a/packager/react-packager/src/Resolver/index.js b/packager/react-packager/src/Resolver/index.js index 707402fac..6bd502c7f 100644 --- a/packager/react-packager/src/Resolver/index.js +++ b/packager/react-packager/src/Resolver/index.js @@ -261,6 +261,10 @@ class Resolver { : result; } + minifyModule({path, code, map}) { + return this._minifyCode(path, code, map); + } + getDebugInfo() { return this._depGraph.getDebugInfo(); } diff --git a/packager/react-packager/src/lib/ModuleTransport.js b/packager/react-packager/src/lib/ModuleTransport.js index 8ba0aaea8..d5e5032a5 100644 --- a/packager/react-packager/src/lib/ModuleTransport.js +++ b/packager/react-packager/src/lib/ModuleTransport.js @@ -22,6 +22,7 @@ function ModuleTransport(data) { this.virtual = data.virtual; this.meta = data.meta; + this.polyfill = data.polyfill; this.map = data.map; Object.freeze(this);