Use the new Graph object for generating delta bundles

Reviewed By: mjesun

Differential Revision: D7275600

fbshipit-source-id: 29579594b88ea19ff81c6e4c1936611f8ecc42f7

packages/metro/src/DeltaBundler/Serializers/Serializers.js

@@ -1,77 +0,0 @@
-/**
- * Copyright (c) 2015-present, Facebook, Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- *
- * @flow
- * @format
- */
-
-'use strict';
-
-import type {BundleOptions} from '../../shared/types.flow';
-import type DeltaBundler from '../';
-import type DeltaTransformer, {
-  DeltaTransformResponse,
-} from '../DeltaTransformer';
-
-export type DeltaOptions = BundleOptions & {
-  deltaBundleId: ?string,
-};
-
-/**
- * This module contains many serializers for the Delta Bundler. Each serializer
- * returns a string representation for any specific type of bundle, which can
- * be directly sent to the devices.
- */
-
-async function deltaBundle(
-  deltaBundler: DeltaBundler,
-  clientId: string,
-  options: DeltaOptions,
-): Promise<{bundle: string, numModifiedFiles: number}> {
-  const {delta} = await _build(deltaBundler, clientId, options);
-
-  function stringifyModule([id, module]) {
-    return [id, module ? module.code : undefined];
-  }
-
-  const bundle = JSON.stringify({
-    id: delta.id,
-    pre: Array.from(delta.pre).map(stringifyModule),
-    post: Array.from(delta.post).map(stringifyModule),
-    delta: Array.from(delta.delta).map(stringifyModule),
-    reset: delta.reset,
-  });
-
-  return {
-    bundle,
-    numModifiedFiles: delta.pre.size + delta.post.size + delta.delta.size,
-  };
-}
-
-async function _build(
-  deltaBundler: DeltaBundler,
-  clientId: string,
-  options: DeltaOptions,
-): Promise<{
-  delta: DeltaTransformResponse,
-  deltaTransformer: DeltaTransformer,
-}> {
-  const deltaTransformer = await deltaBundler.getDeltaTransformer(
-    clientId,
-    options,
-  );
-
-  const delta = await deltaTransformer.getDelta(options.deltaBundleId);
-
-  return {
-    delta,
-    deltaTransformer,
-  };
-}
-
-module.exports = {
-  deltaBundle,
-};

packages/metro/src/DeltaBundler/Serializers/__tests__/Serializers-test.js

@@ -1,88 +0,0 @@
-/**
- * Copyright (c) 2015-present, Facebook, Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- *
- * @emails oncall+javascript_foundation
- * @format
- */
-
-'use strict';
-
-jest.mock('../../../node-haste/lib/toLocalPath');
-jest.mock('../../../Assets');
-
-const toLocalPath = require('../../../node-haste/lib/toLocalPath');
-
-const CURRENT_TIME = 1482363367000;
-
-describe('Serializers', () => {
-  const OriginalDate = global.Date;
-  const getDelta = jest.fn();
-  const getDependenciesFn = jest.fn();
-  const postProcessModules = jest.fn();
-  let deltaBundler;
-  let Serializers;
-
-  const deltaResponse = {
-    id: '1234',
-    pre: new Map([[1, {type: 'script', code: 'pre;', id: 1, path: '/pre.js'}]]),
-    post: new Map([[2, {type: 'require', code: 'post;', id: 2, path: '/p'}]]),
-    delta: new Map([
-      [3, {type: 'module', code: 'module3;', id: 3, path: '/3.js'}],
-      [4, {type: 'module', code: 'another;', id: 4, path: '/4.js'}],
-    ]),
-    inverseDependencies: [],
-    reset: true,
-  };
-
-  function setCurrentTime(time: number) {
-    global.Date = jest.fn(() => new OriginalDate(time));
-  }
-
-  beforeEach(() => {
-    Serializers = require('../Serializers');
-
-    getDelta.mockReturnValueOnce(Promise.resolve(deltaResponse));
-    getDependenciesFn.mockReturnValue(Promise.resolve(() => new Set()));
-    postProcessModules.mockImplementation(modules => modules);
-
-    deltaBundler = {
-      getDeltaTransformer: jest.fn().mockReturnValue(
-        Promise.resolve({
-          getDelta,
-          getDependenciesFn,
-        }),
-      ),
-      getPostProcessModulesFn() {
-        return postProcessModules;
-      },
-    };
-
-    toLocalPath.mockImplementation((roots, path) => path.replace(roots[0], ''));
-
-    setCurrentTime(CURRENT_TIME);
-  });
-
-  it('should return the stringified delta bundle', async () => {
-    expect(
-      await Serializers.deltaBundle(deltaBundler, 'foo', {deltaBundleId: 10}),
-    ).toMatchSnapshot();
-
-    // Simulate a delta with some changes now
-    getDelta.mockReturnValueOnce(
-      Promise.resolve({
-        id: '1234',
-        delta: new Map([[3, {code: 'modified module;'}], [4, null]]),
-        pre: new Map(),
-        post: new Map(),
-        inverseDependencies: [],
-      }),
-    );
-
-    expect(
-      await Serializers.deltaBundle(deltaBundler, 'foo', {deltaBundleId: 10}),
-    ).toMatchSnapshot();
-  });
-});

packages/metro/src/DeltaBundler/Serializers/__tests__/__snapshots__/Serializers-test.js.snap

@@ -1,15 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`Serializers should return the stringified delta bundle 1`] = `
-Object {
-  "bundle": "{\\"id\\":\\"1234\\",\\"pre\\":[[1,\\"pre;\\"]],\\"post\\":[[2,\\"post;\\"]],\\"delta\\":[[3,\\"module3;\\"],[4,\\"another;\\"]],\\"reset\\":true}",
-  "numModifiedFiles": 4,
-}
-`;
-
-exports[`Serializers should return the stringified delta bundle 2`] = `
-Object {
-  "bundle": "{\\"id\\":\\"1234\\",\\"pre\\":[],\\"post\\":[],\\"delta\\":[[3,\\"modified module;\\"],[4,null]]}",
-  "numModifiedFiles": 2,
-}
-`;

packages/metro/src/DeltaBundler/Serializers/__tests__/deltaJSBundle-test.js

@@ -0,0 +1,129 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+javascript_foundation
+ * @format
+ */
+
+'use strict';
+
+const createModuleIdFactory = require('../../../lib/createModuleIdFactory');
+const deltaJSBundle = require('../deltaJSBundle');
+
+function createModule(name, dependencies, type = 'module') {
+  return [
+    `/root/${name}.js`,
+    {
+      path: `/root/${name}.js`,
+      dependencies: new Map(dependencies.map(dep => [dep, `/root/${dep}.js`])),
+      output: {type, code: `__d(function() {${name}()});`},
+    },
+  ];
+}
+
+const prepend = [createModule('prep1', [])[1], createModule('prep2', [])[1]];
+
+const graph = {
+  dependencies: new Map([
+    createModule('entrypoint', ['foo', 'bar']),
+    createModule('foo', []),
+    createModule('bar', []),
+  ]),
+  entryPoints: ['/root/entrypoint.js'],
+};
+
+const options = {
+  createModuleId: createModuleIdFactory(),
+  dev: true,
+  runBeforeMainModule: [],
+  runModule: true,
+  sourceMapUrl: 'http://localhost/bundle.map',
+};
+
+it('returns a reset delta', () => {
+  expect(
+    JSON.parse(
+      deltaJSBundle(
+        'foo',
+        prepend,
+        {
+          modified: graph.dependencies,
+          deleted: new Set(),
+          reset: true,
+        },
+        'sequenceId',
+        graph,
+        options,
+      ),
+    ),
+  ).toEqual({
+    id: 'sequenceId',
+    reset: true,
+    pre: [
+      [-1, '__d(function() {prep1()});'],
+      [-2, '__d(function() {prep2()});'],
+    ],
+    delta: [
+      [0, '__d(function() {entrypoint()},0,[1,2],"entrypoint.js");'],
+      [1, '__d(function() {foo()},1,[],"foo.js");'],
+      [2, '__d(function() {bar()},2,[],"bar.js");'],
+    ],
+    post: [[3, '//# sourceMappingURL=http://localhost/bundle.map']],
+  });
+});
+
+it('returns an incremental delta with modified files', () => {
+  expect(
+    JSON.parse(
+      deltaJSBundle(
+        'foo',
+        prepend,
+        {
+          modified: new Map([createModule('bar', [])]),
+          deleted: new Set(),
+          reset: false,
+        },
+        'sequenceId',
+        graph,
+        options,
+      ),
+    ),
+  ).toEqual({
+    id: 'sequenceId',
+    reset: false,
+    pre: [],
+    post: [],
+    delta: [[2, '__d(function() {bar()},2,[],"bar.js");']],
+  });
+});
+
+it('returns an incremental delta with deleted files', () => {
+  expect(
+    JSON.parse(
+      deltaJSBundle(
+        'foo',
+        prepend,
+        {
+          modified: new Map([createModule('entrypoint', ['foo'])]),
+          deleted: new Set(['/root/bar.js']),
+          reset: false,
+        },
+        'sequenceId',
+        graph,
+        options,
+      ),
+    ),
+  ).toEqual({
+    id: 'sequenceId',
+    reset: false,
+    pre: [],
+    post: [],
+    delta: [
+      [0, '__d(function() {entrypoint()},0,[1],"entrypoint.js");'],
+      [2, null],
+    ],
+  });
+});

packages/metro/src/DeltaBundler/Serializers/deltaJSBundle.js

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ */
+
+'use strict';
+
+const getAppendScripts = require('../../lib/getAppendScripts');
+
+const {wrapModule} = require('./helpers/js');
+
+import type {Delta, Graph} from '../';
+import type {DependencyEdge} from '../traverseDependencies';
+
+type Options = {|
+  createModuleId: string => number | string,
+  +dev: boolean,
+  +runBeforeMainModule: $ReadOnlyArray<string>,
+  +runModule: boolean,
+  +sourceMapUrl: ?string,
+|};
+
+function deltaJSBundle(
+  entryPoint: string,
+  pre: $ReadOnlyArray<DependencyEdge>,
+  delta: Delta,
+  sequenceId: string,
+  graph: Graph,
+  options: Options,
+): string {
+  const outputPre = [];
+  const outputPost = [];
+  const outputDelta = [];
+
+  for (const module of delta.modified.values()) {
+    outputDelta.push([
+      options.createModuleId(module.path),
+      wrapModule(module, options),
+    ]);
+  }
+
+  for (const path of delta.deleted) {
+    outputDelta.push([options.createModuleId(path), null]);
+  }
+
+  if (delta.reset) {
+    let i = -1;
+
+    for (const module of pre) {
+      outputPre.push([i, module.output.code]);
+      i--;
+    }
+
+    const appendScripts = getAppendScripts(entryPoint, graph, options).values();
+
+    for (const module of appendScripts) {
+      outputPost.push([
+        options.createModuleId(module.path),
+        module.output.code,
+      ]);
+    }
+  }
+
+  const output = {
+    id: sequenceId,
+    pre: outputPre,
+    post: outputPost,
+    delta: outputDelta,
+    reset: delta.reset,
+  };
+
+  return JSON.stringify(output);
+}
+
+module.exports = deltaJSBundle;

packages/metro/src/Server/__tests__/Server-test.js

@@ -20,22 +20,21 @@ jest
   .mock('../../Bundler')
   .mock('../../DeltaBundler')
   .mock('../../Assets')
-  .mock('../../lib/getPrependedScripts')
   .mock('../../node-haste/DependencyGraph')
   .mock('metro-core/src/Logger')
   .mock('../../lib/getAbsolutePath')
-  .mock('../../lib/GlobalTransformCache')
+  .mock('../../lib/getPrependedScripts')
-  .mock('../../DeltaBundler/Serializers/Serializers');
+  .mock('../../lib/GlobalTransformCache');
 
 const NativeDate = global.Date;
 
 describe('processRequest', () => {
   let Bundler;
   let Server;
+  let crypto;
   let getAsset;
   let getPrependedScripts;
   let symbolicate;
-  let Serializers;
   let DeltaBundler;
 
   beforeEach(() => {
@@ -46,10 +45,10 @@ describe('processRequest', () => {
 
     Bundler = require('../../Bundler');
     Server = require('../');
+    crypto = require('crypto');
     getAsset = require('../../Assets').getAsset;
     getPrependedScripts = require('../../lib/getPrependedScripts');
     symbolicate = require('../symbolicate/symbolicate');
-    Serializers = require('../../DeltaBundler/Serializers/Serializers');
     DeltaBundler = require('../../DeltaBundler');
   });
 
@@ -148,6 +147,9 @@ describe('processRequest', () => {
 
     server = new Server(options);
     requestHandler = server.processRequest.bind(server);
+
+    let i = 0;
+    crypto.randomBytes.mockImplementation(() => `XXXXX-${i++}`);
   });
 
   it('returns JS bundle source on request of *.bundle', async () => {
@@ -381,46 +383,109 @@ describe('processRequest', () => {
   });
 
   describe('Generate delta bundle endpoint', () => {
-    it('should generate a new delta correctly', () => {
+    it('should generate the initial delta correctly', async () => {
-      Serializers.deltaBundle.mockImplementation(async (_, options) => {
+      const response = await makeRequest(
-        expect(options.deltaBundleId).toBe(undefined);
-
-        return {
-          bundle: '{"delta": "bundle"}',
-          numModifiedFiles: 3,
-        };
-      });
-
-      return makeRequest(requestHandler, 'index.delta?platform=ios').then(
-        function(response) {
-          expect(response.body).toEqual('{"delta": "bundle"}');
-        },
-      );
-    });
-
-    it('should send the correct deltaBundlerId to the bundler', () => {
-      Serializers.deltaBundle.mockImplementation(
-        async (_, clientId, options) => {
-          expect(clientId).toMatchSnapshot();
-          expect(options.deltaBundleId).toBe('1234');
-
-          return {
-            bundle: '{"delta": "bundle"}',
-            numModifiedFiles: 3,
-          };
-        },
-      );
-
-      return makeRequest(
         requestHandler,
-        'index.delta?platform=ios&deltaBundleId=1234',
+        'index.delta?platform=ios',
-      ).then(function(response) {
+      );
-        expect(response.body).toEqual('{"delta": "bundle"}');
+
+      expect(JSON.parse(response.body)).toEqual({
+        id: 'XXXXX-0',
+        pre: [[-1, 'function () {require();}']],
+        delta: [
+          [0, '__d(function() {entry();},0,[1],"mybundle.js");'],
+          [1, '__d(function() {foo();},1,[],"foo.js");'],
+        ],
+        post: [
+          [
+            2,
+            '//# sourceMappingURL=http://localhost:8081/index.map?platform=ios',
+          ],
+        ],
+        reset: true,
+      });
+    });
+
+    it('should generate an incremental delta correctly', async () => {
+      DeltaBundler.prototype.getDelta.mockReturnValue(
+        Promise.resolve({
+          modified: new Map([
+            [
+              '/root/foo.js',
+              {
+                path: '/root/foo.js',
+                output: {code: '__d(function() {modified();});'},
+                dependencies: new Map(),
+              },
+            ],
+          ]),
+          deleted: new Set(),
+          reset: false,
+        }),
+      );
+
+      // initial request.
+      await makeRequest(requestHandler, 'index.delta?platform=ios');
+
+      const response = await makeRequest(
+        requestHandler,
+        'index.delta?platform=ios&deltaBundleId=XXXXX-0',
+      );
+
+      expect(JSON.parse(response.body)).toEqual({
+        id: 'XXXXX-1',
+        pre: [],
+        post: [],
+        delta: [[1, '__d(function() {modified();},1,[],"foo.js");']],
+        reset: false,
+      });
+
+      expect(DeltaBundler.prototype.getDelta.mock.calls[0][1]).toEqual({
+        reset: false,
+      });
+    });
+
+    it('should return a reset delta if the sequenceId does not match', async () => {
+      DeltaBundler.prototype.getDelta.mockReturnValue(
+        Promise.resolve({
+          modified: new Map([
+            [
+              '/root/foo.js',
+              {
+                path: '/root/foo.js',
+                output: {code: '__d(function() {modified();});'},
+                dependencies: new Map(),
+              },
+            ],
+          ]),
+          deleted: new Set(),
+          reset: false,
+        }),
+      );
+
+      // Do an initial request.
+      await makeRequest(requestHandler, 'index.delta?platform=ios');
+      // First delta request has a matching id.
+      await makeRequest(
+        requestHandler,
+        'index.delta?platform=ios&deltaBundleId=XXXXX-0',
+      );
+      // Second delta request does not have a matching id.
+      await makeRequest(
+        requestHandler,
+        'index.delta?platform=ios&deltaBundleId=XXXXX-0',
+      );
+
+      expect(DeltaBundler.prototype.getDelta.mock.calls[0][1]).toEqual({
+        reset: false,
+      });
+      expect(DeltaBundler.prototype.getDelta.mock.calls[1][1]).toEqual({
+        reset: true,
       });
     });
 
     it('should include the error message for transform errors', () => {
-      Serializers.deltaBundle.mockImplementation(async () => {
+      DeltaBundler.prototype.buildGraph.mockImplementation(async () => {
         const transformError = new SyntaxError('test syntax error');
         transformError.type = 'TransformError';
         transformError.filename = 'testFile.js';

packages/metro/src/Server/__tests__/__snapshots__/Server-test.js.snap

@@ -1,3 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`processRequest Generate delta bundle endpoint should send the correct deltaBundlerId to the bundler 1`] = `"{\\"sourceMapUrl\\":null,\\"bundleType\\":null,\\"customTransformOptions\\":{},\\"entryFile\\":\\"/root/index.js\\",\\"deltaBundleId\\":null,\\"dev\\":true,\\"minify\\":false,\\"excludeSource\\":null,\\"hot\\":true,\\"runBeforeMainModule\\":[\\"InitializeCore\\"],\\"runModule\\":true,\\"inlineSourceMap\\":false,\\"isolateModuleIDs\\":false,\\"platform\\":\\"ios\\",\\"resolutionResponse\\":null,\\"entryModuleOnly\\":false,\\"assetPlugins\\":[],\\"onProgress\\":null,\\"unbundle\\":false}"`;

packages/metro/src/Server/index.js

@@ -13,9 +13,10 @@
 const Bundler = require('../Bundler');
 const DeltaBundler = require('../DeltaBundler');
 const MultipartResponse = require('./MultipartResponse');
-const Serializers = require('../DeltaBundler/Serializers/Serializers');
 
+const crypto = require('crypto');
 const defaultCreateModuleIdFactory = require('../lib/createModuleIdFactory');
+const deltaJSBundle = require('../DeltaBundler/Serializers/deltaJSBundle');
 const getAllFiles = require('../DeltaBundler/Serializers/getAllFiles');
 const getAssets = require('../DeltaBundler/Serializers/getAssets');
 const getRamBundleInfo = require('../DeltaBundler/Serializers/getRamBundleInfo');
@@ -44,7 +45,6 @@ import type {CustomError} from '../lib/formatBundlingError';
 import type {DependencyEdge} from '../DeltaBundler/traverseDependencies';
 import type {IncomingMessage, ServerResponse} from 'http';
 import type {Reporter} from '../lib/reporting';
-import type {DeltaOptions} from '../DeltaBundler/Serializers/Serializers';
 import type {RamBundleInfo} from '../DeltaBundler/Serializers/getRamBundleInfo';
 import type {BundleOptions, Options} from '../shared/types.flow';
 import type {
@@ -53,7 +53,7 @@ import type {
   PostProcessBundleSourcemap,
 } from '../Bundler';
 import type {CacheStore} from 'metro-cache';
-import type {Graph} from '../DeltaBundler';
+import type {Delta, Graph} from '../DeltaBundler';
 import type {MetroSourceMap} from 'metro-source-map';
 import type {TransformCache} from '../lib/TransformCaching';
 import type {Symbolicate} from './symbolicate/symbolicate';
@@ -70,8 +70,13 @@ type GraphInfo = {|
   graph: Graph,
   prepend: $ReadOnlyArray<DependencyEdge>,
   lastModified: Date,
+  +sequenceId: string,
 |};
 
+type DeltaOptions = BundleOptions & {
+  deltaBundleId: ?string,
+};
+
 function debounceAndBatch(fn, delay) {
   let timeout;
   return () => {
@@ -127,6 +132,7 @@ class Server {
   _nextBundleBuildID: number;
   _deltaBundler: DeltaBundler;
   _graphs: Map<string, GraphInfo> = new Map();
+  _deltaGraphs: Map<string, GraphInfo> = new Map();
 
   constructor(options: Options) {
     const reporter =
@@ -365,6 +371,7 @@ class Server {
       prepend,
       graph,
       lastModified: new Date(),
+      sequenceId: crypto.randomBytes(8).toString('hex'),
     };
   }
 
@@ -395,6 +402,43 @@ class Server {
     return {...graphInfo, numModifiedFiles};
   }
 
+  async _getDeltaInfo(
+    options: DeltaOptions,
+  ): Promise<{...GraphInfo, delta: Delta}> {
+    const id = this._optionsHash(options);
+    let graphInfo = this._deltaGraphs.get(id);
+
+    let delta;
+
+    if (!graphInfo) {
+      graphInfo = await this._buildGraph(options);
+
+      delta = {
+        modified: graphInfo.graph.dependencies,
+        deleted: new Set(),
+        reset: true,
+      };
+    } else {
+      delta = await this._deltaBundler.getDelta(graphInfo.graph, {
+        reset: graphInfo.sequenceId !== options.deltaBundleId,
+      });
+
+      // Generate a new sequenceId, to be used to verify the next delta request.
+      // $FlowIssue #16581373 spread of an exact object should be exact
+      graphInfo = {
+        ...graphInfo,
+        sequenceId: crypto.randomBytes(8).toString('hex'),
+      };
+    }
+
+    this._deltaGraphs.set(id, graphInfo);
+
+    return {
+      ...graphInfo,
+      delta,
+    };
+  }
+
   async _minifyModule(module: DependencyEdge): Promise<DependencyEdge> {
     const {code, map} = await this._bundler.minifyModule(
       module.path,
@@ -637,14 +681,28 @@ class Server {
 
     let output;
 
-    const clientId = this._optionsHash(options);
-
     try {
-      output = await Serializers.deltaBundle(
+      const {delta, graph, prepend, sequenceId} = await this._getDeltaInfo(
-        this._deltaBundler,
-        clientId,
         options,
       );
+
+      output = {
+        bundle: deltaJSBundle(
+          options.entryFile,
+          prepend,
+          delta,
+          sequenceId,
+          graph,
+          {
+            createModuleId: this._opts.createModuleId,
+            dev: options.dev,
+            runBeforeMainModule: options.runBeforeMainModule,
+            runModule: options.runModule,
+            sourceMapUrl: options.sourceMapUrl,
+          },
+        ),
+        numModifiedFiles: delta.modified.size + delta.deleted.size,
+      };
     } catch (error) {
       this._handleError(mres, this._optionsHash(options), error);