Add Source Maps support to Delta Bundler

Reviewed By: jeanlauliac

Differential Revision: D5793499

fbshipit-source-id: 67e49ed5f5bc9ccae2fb4982cd506fc03259589a

packages/metro-bundler/src/DeltaBundler/DeltaCalculator.js

@@ -21,11 +21,11 @@ import type {BundleOptions} from '../Server';
 import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse';
 import type Module from '../node-haste/Module';
 
-export type DeltaResult = {
+export type DeltaResult = {|
-  modified: Map<string, Module>,
+  +modified: Map<string, Module>,
-  deleted: Set<string>,
+  +deleted: Set<string>,
-  reset?: boolean,
+  +reset: boolean,
-};
+|};
 
 /**
  * This class is in charge of calculating the delta of changed modules that
@@ -199,7 +199,7 @@ class DeltaCalculator extends EventEmitter {
 
     // No changes happened. Return empty delta.
     if (modifiedArray.length === 0) {
-      return {modified: new Map(), deleted: new Set()};
+      return {modified: new Map(), deleted: new Set(), reset: false};
     }
 
     // Build the modules from the files that have been modified.
@@ -219,7 +219,7 @@ class DeltaCalculator extends EventEmitter {
     // If there is no file with changes in its dependencies, we can just
     // return the modified modules without recalculating the dependencies.
     if (!filesWithChangedDependencies.some(value => value)) {
-      return {modified, deleted: new Set()};
+      return {modified, deleted: new Set(), reset: false};
     }
 
     // Recalculate all dependencies and append the newly added files to the
@@ -233,6 +233,7 @@ class DeltaCalculator extends EventEmitter {
     return {
       modified,
       deleted,
+      reset: false,
     };
   }
 

packages/metro-bundler/src/DeltaBundler/DeltaPatcher.js

@@ -12,6 +12,8 @@
 
 'use strict';
 
+const {fromRawMappings} = require('../Bundler/source-map');
+
 import type {DeltaBundle} from './';
 
 /**
@@ -21,16 +23,16 @@ import type {DeltaBundle} from './';
  */
 class DeltaPatcher {
   _lastBundle = {
-    pre: '',
+    pre: new Map(),
-    post: '',
+    post: new Map(),
-    modules: {},
+    modules: new Map(),
   };
   _initialized = false;
 
   /**
    * Applies a Delta Bundle to the current bundle.
    */
-  applyDelta(deltaBundle: DeltaBundle) {
+  applyDelta(deltaBundle: DeltaBundle): DeltaPatcher {
     // Make sure that the first received delta is a fresh one.
     if (!this._initialized && !deltaBundle.reset) {
       throw new Error(
@@ -43,30 +45,15 @@ class DeltaPatcher {
     // Reset the current delta when we receive a fresh delta.
     if (deltaBundle.reset) {
       this._lastBundle = {
-        pre: '',
+        pre: new Map(),
-        post: '',
+        post: new Map(),
-        modules: {},
+        modules: new Map(),
       };
     }
 
-    // Override the prepended sources.
+    this._patchMap(this._lastBundle.pre, deltaBundle.pre);
-    if (deltaBundle.pre) {
+    this._patchMap(this._lastBundle.post, deltaBundle.post);
-      this._lastBundle.pre = deltaBundle.pre;
+    this._patchMap(this._lastBundle.modules, deltaBundle.delta);
-    }
-
-    // Override the appended sources.
-    if (deltaBundle.post) {
-      this._lastBundle.post = deltaBundle.post;
-    }
-
-    // Patch the received modules.
-    for (const i in deltaBundle.delta) {
-      if (deltaBundle.delta[i] == null) {
-        delete this._lastBundle.modules[i];
-      } else {
-        this._lastBundle.modules[i] = deltaBundle.delta[i];
-      }
-    }
 
     return this;
   }
@@ -75,14 +62,34 @@ class DeltaPatcher {
    * Converts the current delta bundle to a standard string bundle, ready to
    * be interpreted by any JS VM.
    */
-  stringify() {
+  stringifyCode() {
-    return []
+    const code = this._getAllModules().map(m => m.code);
-      .concat(
+
-        this._lastBundle.pre,
+    return code.join('\n;');
-        Object.values(this._lastBundle.modules),
+  }
-        this._lastBundle.post,
+
-      )
+  stringifyMap({excludeSource}: {excludeSource?: boolean}) {
-      .join('\n;');
+    const mappings = fromRawMappings(this._getAllModules());
+
+    return mappings.toString(undefined, {excludeSource});
+  }
+
+  _getAllModules() {
+    return [].concat(
+      Array.from(this._lastBundle.pre.values()),
+      Array.from(this._lastBundle.modules.values()),
+      Array.from(this._lastBundle.post.values()),
+    );
+  }
+
+  _patchMap<K, V>(original: Map<K, V>, patch: Map<K, ?V>) {
+    for (const [key, value] of patch.entries()) {
+      if (value == null) {
+        original.delete(key);
+      } else {
+        original.set(key, value);
+      }
+    }
   }
 }
 

packages/metro-bundler/src/DeltaBundler/DeltaTransformer.js

@@ -16,6 +16,7 @@ const DeltaCalculator = require('./DeltaCalculator');
 
 const {EventEmitter} = require('events');
 
+import type {RawMapping} from '../Bundler/source-map';
 import type Bundler from '../Bundler';
 import type {Options as JSTransformerOptions} from '../JSTransformer/worker';
 import type Resolver from '../Resolver';
@@ -23,12 +24,23 @@ import type {MappingsMap} from '../lib/SourceMap';
 import type Module from '../node-haste/Module';
 import type {Options as BundleOptions} from './';
 
-export type DeltaTransformResponse = {
+type DeltaEntry = {|
-  +pre: ?string,
+  +code: string,
-  +post: ?string,
+  +map: ?Array<RawMapping>,
-  +delta: {[key: string]: ?string},
+  +name: string,
+  +path: string,
+  +source: string,
+|};
+
+export type DeltaEntries = Map<number, ?DeltaEntry>;
+
+export type DeltaTransformResponse = {|
+  +pre: DeltaEntries,
+  +post: DeltaEntries,
+  +delta: DeltaEntries,
   +inverseDependencies: {[key: string]: $ReadOnlyArray<string>},
-};
+  +reset: boolean,
+|};
 
 type Options = {|
   +getPolyfills: ({platform: ?string}) => $ReadOnlyArray<string>,
@@ -37,18 +49,19 @@ type Options = {|
 
 /**
  * This class is in charge of creating the delta bundle with the actual
- * transformed source code for each of the modified modules.
+ * transformed source code for each of the modified modules. For each modified
+ * module it returns a `DeltaModule` object that contains the basic information
+ * about that file. Modules that have been deleted contain a `null` module
+ * parameter.
  *
- * The delta bundle format is the following:
+ * The actual return format is the following:
  *
  *   {
- *     pre: '...',   // source code to be prepended before all the modules.
+ *     pre: [{id, module: {}}],   Scripts to be prepended before the actual
- *     post: '...',  // source code to be appended after all the modules
+ *                                modules.
- *                   // (normally here lay the require() call for the starup).
+ *     post: [{id, module: {}}],  Scripts to be appended after all the modules
- *     delta: {
+ *                                (normally the initial require() calls).
- *       27: '...',  // transformed source code of a modified module.
+ *     delta: [{id, module: {}}], Actual bundle modules (dependencies).
- *       56: null,   // deleted module.
- *     },
  *   }
  */
 class DeltaTransformer extends EventEmitter {
@@ -142,22 +155,21 @@ class DeltaTransformer extends EventEmitter {
 
     // Get the transformed source code of each modified/added module.
     const modifiedDelta = await this._transformModules(
-      modified,
+      Array.from(modified.values()),
       resolver,
       transformerOptions,
       dependencyPairs,
     );
 
-    const deletedDelta = Object.create(null);
     deleted.forEach(id => {
-      deletedDelta[this._getModuleId({path: id})] = null;
+      modifiedDelta.set(this._getModuleId({path: id}), null);
     });
 
     // Return the source code that gets prepended to all the modules. This
     // contains polyfills and startup code (like the require() implementation).
     const prependSources = reset
       ? await this._getPrepend(transformerOptions, dependencyPairs)
-      : null;
+      : new Map();
 
     // Return the source code that gets appended to all the modules. This
     // contains the require() calls to startup the execution of the modules.
@@ -166,7 +178,7 @@ class DeltaTransformer extends EventEmitter {
           dependencyPairs,
           this._deltaCalculator.getModulesByName(),
         )
-      : null;
+      : new Map();
 
     // Inverse dependencies are needed for HMR.
     const inverseDependencies = this._getInverseDependencies(
@@ -176,7 +188,7 @@ class DeltaTransformer extends EventEmitter {
     return {
       pre: prependSources,
       post: appendSources,
-      delta: {...modifiedDelta, ...deletedDelta},
+      delta: modifiedDelta,
       inverseDependencies,
       reset,
     };
@@ -185,7 +197,7 @@ class DeltaTransformer extends EventEmitter {
   async _getPrepend(
     transformOptions: JSTransformerOptions,
     dependencyPairs: Map<string, $ReadOnlyArray<[string, Module]>>,
-  ): Promise<string> {
+  ): Promise<DeltaEntries> {
     const resolver = await this._bundler.getResolver();
 
     // Get all the polyfills from the relevant option params (the
@@ -210,25 +222,18 @@ class DeltaTransformer extends EventEmitter {
       ),
     );
 
-    const sources = await Promise.all(
+    return await this._transformModules(
-      modules.map(async module => {
+      modules,
-        const result = await this._transformModule(
+      resolver,
-          module,
+      transformOptions,
-          resolver,
+      dependencyPairs,
-          transformOptions,
-          dependencyPairs,
-        );
-        return result[1];
-      }),
     );
-
-    return sources.join('\n;');
   }
 
   async _getAppend(
     dependencyPairs: Map<string, $ReadOnlyArray<[string, Module]>>,
     modulesByName: Map<string, Module>,
-  ): Promise<string> {
+  ): Promise<DeltaEntries> {
     const resolver = await this._bundler.getResolver();
 
     // Get the absolute path of the entry file, in order to be able to get the
@@ -242,14 +247,29 @@ class DeltaTransformer extends EventEmitter {
     // First, get the modules correspondant to all the module names defined in
     // the `runBeforeMainModule` config variable. Then, append the entry point
     // module so the last thing that gets required is the entry point.
-    const sources = this._bundleOptions.runBeforeMainModule
+    return new Map(
-      .map(name => modulesByName.get(name))
+      this._bundleOptions.runBeforeMainModule
-      .concat(entryPointModule)
+        .map(name => modulesByName.get(name))
-      .filter(Boolean)
+        .concat(entryPointModule)
-      .map(this._getModuleId)
+        .filter(Boolean)
-      .map(moduleId => `;require(${JSON.stringify(moduleId)});`);
+        .map(this._getModuleId)
+        .map(moduleId => {
+          const code = `;require(${JSON.stringify(moduleId)});`;
+          const name = 'require-' + String(moduleId);
+          const path = name + '.js';
 
-    return sources.join('\n');
+          return [
+            moduleId,
+            {
+              code,
+              map: null,
+              name,
+              source: code,
+              path,
+            },
+          ];
+        }),
+    );
   }
 
   /**
@@ -270,28 +290,23 @@ class DeltaTransformer extends EventEmitter {
   }
 
   async _transformModules(
-    modules: Map<string, Module>,
+    modules: Array<Module>,
     resolver: Resolver,
     transformOptions: JSTransformerOptions,
     dependencyPairs: Map<string, $ReadOnlyArray<[string, Module]>>,
-  ): Promise<{[key: string]: string}> {
+  ): Promise<DeltaEntries> {
-    const transformedModules = await Promise.all(
+    return new Map(
-      Array.from(modules.values()).map(module =>
+      await Promise.all(
-        this._transformModule(
+        modules.map(module =>
-          module,
+          this._transformModule(
-          resolver,
+            module,
-          transformOptions,
+            resolver,
-          dependencyPairs,
+            transformOptions,
+            dependencyPairs,
+          ),
         ),
       ),
     );
-
-    const output = Object.create(null);
-    transformedModules.forEach(([id, source]) => {
-      output[id] = source;
-    });
-
-    return output;
   }
 
   async _transformModule(
@@ -299,7 +314,7 @@ class DeltaTransformer extends EventEmitter {
     resolver: Resolver,
     transformOptions: JSTransformerOptions,
     dependencyPairs: Map<string, $ReadOnlyArray<[string, Module]>>,
-  ): Promise<[number, string]> {
+  ): Promise<[number, ?DeltaEntry]> {
     const [name, metadata] = await Promise.all([
       module.getName(),
       this._getMetadata(module, transformOptions),
@@ -327,9 +342,25 @@ class DeltaTransformer extends EventEmitter {
             dependencyPairsForModule,
             metadata.dependencyOffsets || [],
           ),
+          map: metadata.map,
         };
 
-    return [this._getModuleId(module), wrapped.code];
+    // Ignore the Source Maps if the output of the transformer is not our
+    // custom rawMapping data structure, since the Delta bundler cannot process
+    // them. This can potentially happen when the minifier is enabled (since
+    // uglifyJS only returns standard Source Maps).
+    const map = Array.isArray(wrapped.map) ? wrapped.map : undefined;
+
+    return [
+      this._getModuleId(module),
+      {
+        code: wrapped.code,
+        map,
+        name,
+        source: metadata.source,
+        path: module.path,
+      },
+    ];
   }
 
   async _getMetadata(
@@ -338,7 +369,8 @@ class DeltaTransformer extends EventEmitter {
   ): Promise<{
     +code: string,
     +dependencyOffsets: ?Array<number>,
-    +map?: ?MappingsMap,
+    +map: ?MappingsMap,
+    +source: string,
   }> {
     if (module.isAsset()) {
       const asset = await this._bundler.generateAssetObjAndCode(
@@ -351,6 +383,7 @@ class DeltaTransformer extends EventEmitter {
         code: asset.code,
         dependencyOffsets: asset.meta.dependencyOffsets,
         map: undefined,
+        source: '',
       };
     }
 

packages/metro-bundler/src/DeltaBundler/__tests__/DeltaCalculator-test.js

@@ -137,6 +137,7 @@ describe('DeltaCalculator', () => {
     expect(await deltaCalculator.getDelta()).toEqual({
       modified: new Map(),
       deleted: new Set(),
+      reset: false,
     });
   });
 
@@ -150,6 +151,7 @@ describe('DeltaCalculator', () => {
     expect(result).toEqual({
       modified: new Map([['/foo', moduleFoo]]),
       deleted: new Set(),
+      reset: false,
     });
   });
 
@@ -167,6 +169,7 @@ describe('DeltaCalculator', () => {
     expect(result).toEqual({
       modified: new Map([['/foo', moduleFoo]]),
       deleted: new Set(['/bar']),
+      reset: false,
     });
   });
 
@@ -185,6 +188,7 @@ describe('DeltaCalculator', () => {
     expect(result).toEqual({
       modified: new Map([['/foo', moduleFoo], ['/qux', moduleQux]]),
       deleted: new Set(['/bar', '/baz']),
+      reset: false,
     });
   });
 

packages/metro-bundler/src/DeltaBundler/__tests__/DeltaPatcher-test.js

@@ -24,9 +24,9 @@ describe('DeltaPatcher', () => {
   it('should throw if received a non-reset delta as the initial one', () => {
     expect(() =>
       deltaPatcher.applyDelta({
-        pre: 'pre',
+        pre: new Map(),
-        post: 'post',
+        post: new Map(),
-        delta: {},
+        delta: new Map(),
       }),
     ).toThrow();
   });
@@ -35,13 +35,11 @@ describe('DeltaPatcher', () => {
     const result = deltaPatcher
       .applyDelta({
         reset: 1,
-        pre: 'pre',
+        pre: new Map([[1, {code: 'pre'}]]),
-        post: 'post',
+        post: new Map([[2, {code: 'post'}]]),
-        delta: {
+        delta: new Map([[3, {code: 'middle'}]]),
-          1: 'middle',
-        },
       })
-      .stringify();
+      .stringifyCode();
 
     expect(result).toMatchSnapshot();
   });
@@ -50,56 +48,48 @@ describe('DeltaPatcher', () => {
     const result = deltaPatcher
       .applyDelta({
         reset: 1,
-        pre: 'pre',
+        pre: new Map([[1000, {code: 'pre'}]]),
-        post: 'post',
+        post: new Map([[2000, {code: 'post'}]]),
-        delta: {
+        delta: new Map([[1, {code: 'middle'}]]),
-          1: 'middle',
-        },
       })
       .applyDelta({
-        delta: {
+        pre: new Map(),
-          2: 'another',
+        post: new Map(),
-        },
+        delta: new Map([[2, {code: 'another'}]]),
       })
       .applyDelta({
-        delta: {
+        pre: new Map(),
-          2: 'another',
+        post: new Map(),
-          87: 'third',
+        delta: new Map([[2, {code: 'another'}], [87, {code: 'third'}]]),
-        },
       })
-      .stringify();
+      .stringifyCode();
 
     expect(result).toMatchSnapshot();
 
     const anotherResult = deltaPatcher
       .applyDelta({
-        pre: 'new pre',
+        pre: new Map([[1000, {code: 'new pre'}]]),
-        delta: {
+        post: new Map(),
-          2: 'another',
+        delta: new Map([[2, {code: 'another'}], [1, null]]),
-          1: null,
-        },
       })
       .applyDelta({
-        delta: {
+        pre: new Map(),
-          2: null,
+        post: new Map(),
-          12: 'twelve',
+        delta: new Map([[2, null], [12, {code: 'twelve'}]]),
-        },
       })
-      .stringify();
+      .stringifyCode();
 
     expect(anotherResult).toMatchSnapshot();
 
     expect(
       deltaPatcher
         .applyDelta({
-          pre: '1',
+          pre: new Map([[1000, {code: '1'}]]),
-          post: '1',
+          post: new Map([[1000, {code: '1'}]]),
-          delta: {
+          delta: new Map([[12, {code: 'ten'}]]),
-            12: 'ten',
-          },
           reset: true,
         })
-        .stringify(),
+        .stringifyCode(),
     ).toMatchSnapshot();
   });
 });

packages/metro-bundler/src/DeltaBundler/__tests__/__snapshots__/DeltaPatcher-test.js.snap

@@ -16,8 +16,8 @@ exports[`DeltaPatcher should apply many different patches correctly 1`] = `
 
 exports[`DeltaPatcher should apply many different patches correctly 2`] = `
 "new pre
-;twelve
 ;third
+;twelve
 ;post"
 `;
 

packages/metro-bundler/src/DeltaBundler/index.js

@@ -17,14 +17,16 @@ const DeltaTransformer = require('./DeltaTransformer');
 
 import type Bundler from '../Bundler';
 import type {BundleOptions} from '../Server';
+import type {DeltaEntries} from './DeltaTransformer';
 
-export type DeltaBundle = {
+export type DeltaBundle = {|
-  id: string,
+  +id: string,
-  pre: ?string,
+  +pre: DeltaEntries,
-  post: ?string,
+  +post: DeltaEntries,
-  delta: {[key: string]: ?string},
+  +delta: DeltaEntries,
-  inverseDependencies: {[key: string]: $ReadOnlyArray<string>},
+  +inverseDependencies: {[key: string]: $ReadOnlyArray<string>},
-};
+  +reset: boolean,
+|};
 
 type MainOptions = {|
   getPolyfills: ({platform: ?string}) => $ReadOnlyArray<string>,
@@ -59,7 +61,12 @@ class DeltaBundler {
   }
 
   async build(options: Options): Promise<DeltaBundle> {
-    const {deltaTransformer, id} = await this.getDeltaTransformer(options);
+    const {deltaTransformer, id} = await this.getDeltaTransformer({
+      ...options,
+      // The Delta Bundler does not support minifying due to issues generating
+      // the source maps (T21699790).
+      minify: false,
+    });
     const response = await deltaTransformer.getDelta();
 
     return {
@@ -99,6 +106,22 @@ class DeltaBundler {
   }
 
   async buildFullBundle(options: FullBuildOptions): Promise<string> {
+    let output = (await this._getDeltaPatcher(options)).stringifyCode();
+
+    if (options.sourceMapUrl) {
+      output += '//# sourceMappingURL=' + options.sourceMapUrl;
+    }
+
+    return output;
+  }
+
+  async buildFullSourceMap(options: FullBuildOptions): Promise<string> {
+    return (await this._getDeltaPatcher(options)).stringifyMap({
+      excludeSource: options.excludeSource,
+    });
+  }
+
+  async _getDeltaPatcher(options: FullBuildOptions): Promise<DeltaPatcher> {
     const deltaBundle = await this.build({
       ...options,
       wrapModules: true,
@@ -108,11 +131,10 @@ class DeltaBundler {
 
     if (!deltaPatcher) {
       deltaPatcher = new DeltaPatcher();
-
       this._deltaPatchers.set(deltaBundle.id, deltaPatcher);
     }
 
-    return deltaPatcher.applyDelta(deltaBundle).stringify();
+    return deltaPatcher.applyDelta(deltaBundle);
   }
 }
 

packages/metro-bundler/src/HmrServer/index.js

@@ -95,11 +95,11 @@ class HmrServer<TClient: Client> {
     const result = await client.deltaTransformer.getDelta();
     const modules = [];
 
-    for (const id in result.delta) {
+    for (const [id, module] of result.delta) {
       // The Delta Bundle can have null objects: these correspond to deleted
       // modules, which we don't need to send to the client.
-      if (result.delta[id] != null) {
+      if (module != null) {
-        modules.push({id, code: result.delta[id]});
+        modules.push({id, code: module.code});
       }
     }