Add module for incrementally traversing the dependencies

Reviewed By: davidaurelio

Differential Revision: D5880827

fbshipit-source-id: 9500c54c2f93726449a413321b9e4ef46825b977

packages/metro-bundler/src/DeltaBundler/__tests__/traverseDependencies-test.js

@@ -0,0 +1,192 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @emails oncall+javascript_foundation
+ * @format
+ */
+
+'use strict';
+
+const {
+  initialTraverseDependencies,
+  traverseDependencies,
+} = require('../traverseDependencies');
+
+const entryModule = createModule({path: '/bundle', name: 'bundle'});
+const moduleFoo = createModule({path: '/foo', name: 'foo'});
+const moduleBar = createModule({path: '/bar', name: 'bar'});
+const moduleBaz = createModule({path: '/baz', name: 'baz'});
+
+let dependencyGraph;
+let mockedDependencies;
+let mockedDependencyTree;
+
+function createModule({path, name, isAsset, isJSON}) {
+  return {
+    path,
+    name,
+    async getName() {
+      return name;
+    },
+    isAsset() {
+      return !!isAsset;
+    },
+    isJSON() {
+      return !!isAsset;
+    },
+  };
+}
+
+beforeEach(async () => {
+  mockedDependencies = new Set([entryModule, moduleFoo, moduleBar, moduleBaz]);
+  mockedDependencyTree = new Map([
+    [entryModule.path, [moduleFoo]],
+    [moduleFoo.path, [moduleBar, moduleBaz]],
+  ]);
+
+  dependencyGraph = {
+    getAbsolutePath(path) {
+      return '/' + path;
+    },
+    getModuleForPath(path) {
+      return Array.from(mockedDependencies).find(dep => dep.path === path);
+    },
+    async getShallowDependencies(path) {
+      const deps = mockedDependencyTree.get(path);
+      return deps ? await Promise.all(deps.map(dep => dep.getName())) : [];
+    },
+    resolveDependency(module, relativePath) {
+      const deps = mockedDependencyTree.get(module.path);
+      const dependency = deps.filter(dep => dep.name === relativePath)[0];
+
+      if (!mockedDependencies.has(dependency)) {
+        throw new Error('Dependency not found');
+      }
+      return dependency;
+    },
+  };
+});
+
+it('should do the initial traversal correctly', async () => {
+  const edges = new Map();
+  const result = await initialTraverseDependencies(
+    '/bundle',
+    dependencyGraph,
+    {},
+    edges,
+  );
+
+  expect(result).toEqual({
+    added: new Set(['/foo', '/bar', '/baz']),
+    deleted: new Set(),
+  });
+});
+
+it('should return an empty result when there are no changes', async () => {
+  const edges = new Map();
+  await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+  expect(
+    await traverseDependencies(['/bundle'], dependencyGraph, {}, edges),
+  ).toEqual({
+    added: new Set(),
+    deleted: new Set(),
+  });
+});
+
+it('should return a removed dependency', async () => {
+  const edges = new Map();
+  await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+  // Remove moduleBar
+  mockedDependencyTree.set(moduleFoo.path, [moduleBaz]);
+
+  expect(
+    await traverseDependencies(['/foo'], dependencyGraph, {}, edges),
+  ).toEqual({
+    added: new Set(),
+    deleted: new Set(['/bar']),
+  });
+});
+
+it('should return added/removed dependencies', async () => {
+  const edges = new Map();
+  await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+  // Add moduleQux
+  const moduleQux = createModule({path: '/qux', name: 'qux'});
+  mockedDependencyTree.set(moduleFoo.path, [moduleQux]);
+  mockedDependencies.add(moduleQux);
+
+  expect(
+    await traverseDependencies(['/foo'], dependencyGraph, {}, edges),
+  ).toEqual({
+    added: new Set(['/qux']),
+    deleted: new Set(['/bar', '/baz']),
+  });
+});
+
+it('should retry to traverse the dependencies as it was after getting an error', async () => {
+  const edges = new Map();
+  await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+  mockedDependencies.delete(moduleBar);
+
+  await expect(
+    traverseDependencies(['/foo'], dependencyGraph, {}, edges),
+  ).rejects.toBeInstanceOf(Error);
+
+  // Second time that the traversal of dependencies we still have to throw an
+  // error (no matter if no file has been changed).
+  await expect(
+    traverseDependencies(['/foo'], dependencyGraph, {}, edges),
+  ).rejects.toBeInstanceOf(Error);
+});
+
+describe('edge cases', () => {
+  it('should handle renames correctly', async () => {
+    const edges = new Map();
+    await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+    // Change the dependencies of /path, removing /baz and adding /qux.
+    const moduleQux = createModule({path: '/qux', name: 'qux'});
+    mockedDependencyTree.set(moduleFoo.path, [moduleQux, moduleBar]);
+    mockedDependencies.add(moduleQux);
+    mockedDependencies.delete(moduleBaz);
+
+    // Call traverseDependencies with /foo, /qux and /baz, simulating that the
+    // user has modified the 3 files.
+    expect(
+      await traverseDependencies(
+        ['/foo', '/qux', '/baz'],
+        dependencyGraph,
+        {},
+        edges,
+      ),
+    ).toEqual({
+      added: new Set(['/qux']),
+      deleted: new Set(['/baz']),
+    });
+  });
+
+  it('modify a file and delete it afterwards', async () => {
+    const edges = new Map();
+    await initialTraverseDependencies('/bundle', dependencyGraph, {}, edges);
+
+    mockedDependencyTree.set(moduleFoo.path, [moduleBar]);
+    mockedDependencies.delete(moduleBaz);
+
+    // Modify /baz, rename it to /qux and modify it again.
+    expect(
+      await traverseDependencies(['/baz', '/foo'], dependencyGraph, {}, edges),
+    ).toEqual({
+      added: new Set(),
+      deleted: new Set(['/baz']),
+    });
+  });
+});

packages/metro-bundler/src/DeltaBundler/traverseDependencies.js

@@ -0,0 +1,346 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ * @format
+ */
+
+'use strict';
+
+import type {Options as JSTransformerOptions} from '../JSTransformer/worker';
+import type DependencyGraph from '../node-haste/DependencyGraph';
+import type {
+  InverseDependencies,
+  ModulePaths,
+  ShallowDependencies,
+} from './DeltaCalculator';
+
+export type Caches = {|
+  inverseDependencies: InverseDependencies,
+  modules: ModulePaths,
+  shallowDependencies: ShallowDependencies,
+|};
+
+export type DependencyEdge = {|
+  dependencies: Map<string, string>,
+  inverseDependencies: Set<string>,
+  path: string,
+|};
+
+export type DependencyEdges = Map<string, DependencyEdge>;
+
+type Result = {added: Set<string>, deleted: Set<string>};
+
+/**
+ * Dependency Traversal logic for the Delta Bundler. This method calculates
+ * the modules that should be included in the bundle by traversing the
+ * dependency graph.
+ * Instead of traversing the whole graph each time, it just calculates the
+ * difference between runs by only traversing the added/removed dependencies.
+ * To do so, it uses the passed `edges` paramater, which is a data structure
+ * that contains the whole status of the dependency graph. During the
+ * recalculation of the dependencies, it mutates the edges graph.
+ *
+ * The paths parameter contains the absolute paths of the root files that the
+ * method should traverse. Normally, these paths should be the modified files
+ * since the last traversal.
+ */
+async function traverseDependencies(
+  paths: Array<string>,
+  dependencyGraph: DependencyGraph,
+  transformOptions: JSTransformerOptions,
+  edges: DependencyEdges,
+  onProgress?: (numProcessed: number, total: number) => mixed = () => {},
+): Promise<Result> {
+  const changes = await Promise.all(
+    paths.map(path =>
+      traverseDependenciesForSingleFile(
+        path,
+        dependencyGraph,
+        transformOptions,
+        edges,
+        onProgress,
+      ),
+    ),
+  );
+
+  const added = new Set();
+  const deleted = new Set();
+
+  for (const change of changes) {
+    for (const path of change.added) {
+      added.add(path);
+    }
+    for (const path of change.deleted) {
+      deleted.add(path);
+    }
+  }
+
+  return {
+    added,
+    deleted,
+  };
+}
+
+async function initialTraverseDependencies(
+  path: string,
+  dependencyGraph: DependencyGraph,
+  transformOptions: JSTransformerOptions,
+  edges: DependencyEdges,
+  onProgress?: (numProcessed: number, total: number) => mixed = () => {},
+) {
+  createEdge(path, edges);
+
+  return await traverseDependenciesForSingleFile(
+    path,
+    dependencyGraph,
+    transformOptions,
+    edges,
+    onProgress,
+  );
+}
+
+async function traverseDependenciesForSingleFile(
+  path: string,
+  dependencyGraph: DependencyGraph,
+  transformOptions: JSTransformerOptions,
+  edges: DependencyEdges,
+  onProgress?: (numProcessed: number, total: number) => mixed = () => {},
+): Promise<Result> {
+  const edge = edges.get(path);
+
+  // If the passed edge does not exist does not exist in the graph, ignore it.
+  if (!edge) {
+    return {added: new Set(), deleted: new Set()};
+  }
+
+  const currentDependencies = new Set(
+    await dependencyGraph.getShallowDependencies(path, transformOptions),
+  );
+  const previousDependencies = new Set(edge.dependencies.keys());
+
+  const nonNullEdge = edge;
+
+  let numProcessed = 0;
+  let total = currentDependencies.size;
+
+  // Check all the module dependencies and start traversing the tree from each
+  // added and removed dependency, to get all the modules that have to be added
+  // and removed from the dependency graph.
+  const added = await Promise.all(
+    Array.from(currentDependencies).map(async dependency => {
+      let newDependencies;
+
+      if (!previousDependencies.has(dependency)) {
+        newDependencies = await addDependency(
+          nonNullEdge,
+          dependency,
+          dependencyGraph,
+          transformOptions,
+          edges,
+        );
+      } else {
+        newDependencies = new Set();
+      }
+
+      numProcessed += newDependencies.size + 1;
+      total += newDependencies.size;
+      onProgress(numProcessed, total);
+
+      return newDependencies;
+    }),
+  );
+
+  // Check if all currentDependencies are still in the bundle (some files can
+  // have been removed).
+  checkModuleDependencies(
+    path,
+    currentDependencies,
+    dependencyGraph,
+    transformOptions,
+    edges,
+  );
+
+  const deleted = Array.from(previousDependencies)
+    .map(dependency => {
+      if (!currentDependencies.has(dependency)) {
+        return removeDependency(nonNullEdge, dependency, edges);
+      } else {
+        return undefined;
+      }
+    })
+    .filter(Boolean);
+
+  return {
+    added: flatten(added),
+    deleted: flatten(deleted),
+  };
+}
+
+async function addDependency(
+  parentEdge: DependencyEdge,
+  relativePath: string,
+  dependencyGraph: DependencyGraph,
+  transformOptions: JSTransformerOptions,
+  edges: DependencyEdges,
+): Promise<Set<string>> {
+  const parentModule = dependencyGraph.getModuleForPath(parentEdge.path);
+  const module = dependencyGraph.resolveDependency(
+    parentModule,
+    relativePath,
+    transformOptions.platform,
+  );
+
+  // Update the parent edge to keep track of the new dependency.
+  parentEdge.dependencies.set(relativePath, module.path);
+
+  let dependencyEdge = edges.get(module.path);
+
+  // The new dependency was already in the graph, we don't need to do anything.
+  if (dependencyEdge) {
+    dependencyEdge.inverseDependencies.add(parentEdge.path);
+
+    return new Set();
+  }
+
+  // Create the new edge and traverse all its subdependencies, looking for new
+  // subdependencies recursively.
+  dependencyEdge = createEdge(module.path, edges);
+  dependencyEdge.inverseDependencies.add(parentEdge.path);
+
+  const addedDependencies = new Set([dependencyEdge.path]);
+
+  const shallowDeps = await dependencyGraph.getShallowDependencies(
+    dependencyEdge.path,
+    transformOptions,
+  );
+
+  const nonNullDependencyEdge = dependencyEdge;
+
+  const added = await Promise.all(
+    shallowDeps.map(dep =>
+      addDependency(
+        nonNullDependencyEdge,
+        dep,
+        dependencyGraph,
+        transformOptions,
+        edges,
+      ),
+    ),
+  );
+
+  for (const newDependency of flatten(added)) {
+    addedDependencies.add(newDependency);
+  }
+
+  return addedDependencies;
+}
+
+function removeDependency(
+  parentEdge: DependencyEdge,
+  relativePath: string,
+  edges: DependencyEdges,
+): Set<string> {
+  // Find the actual edge that represents the removed dependency. We do this
+  // from the egdes data structure, since the file may have been deleted
+  // already.
+  const dependencyEdge = resolveEdge(parentEdge, relativePath, edges);
+  if (!dependencyEdge) {
+    return new Set();
+  }
+
+  parentEdge.dependencies.delete(relativePath);
+  dependencyEdge.inverseDependencies.delete(parentEdge.path);
+
+  // This module is still used by another modules, so we cannot remove it from
+  // the bundle.
+  if (dependencyEdge.inverseDependencies.size) {
+    return new Set();
+  }
+
+  const removedDependencies = new Set([dependencyEdge.path]);
+
+  // Now we need to iterate through the module dependencies in order to
+  // clean up everything (we cannot read the module because it may have
+  // been deleted).
+  for (const subDependency of dependencyEdge.dependencies.keys()) {
+    const removed = removeDependency(dependencyEdge, subDependency, edges);
+
+    for (const removedDependency of removed.values()) {
+      removedDependencies.add(removedDependency);
+    }
+  }
+
+  // This module is not used anywhere else!! we can clear it from the bundle
+  destroyEdge(dependencyEdge, edges);
+
+  return removedDependencies;
+}
+
+function createEdge(path: string, edges: DependencyEdges): DependencyEdge {
+  const edge = {
+    dependencies: new Map(),
+    inverseDependencies: new Set(),
+    path,
+  };
+  edges.set(path, edge);
+
+  return edge;
+}
+
+function destroyEdge(edge: DependencyEdge, edges: DependencyEdges) {
+  edges.delete(edge.path);
+}
+
+function resolveEdge(
+  parentEdge: DependencyEdge,
+  relativePath: string,
+  edges: DependencyEdges,
+): ?DependencyEdge {
+  const absolutePath = parentEdge.dependencies.get(relativePath);
+  if (!absolutePath) {
+    return null;
+  }
+
+  return edges.get(absolutePath);
+}
+
+function checkModuleDependencies(
+  parentPath,
+  dependencies: Set<string>,
+  dependencyGraph: DependencyGraph,
+  transformOptions: JSTransformerOptions,
+  edges: DependencyEdges,
+) {
+  const parentModule = dependencyGraph.getModuleForPath(parentPath);
+
+  for (const dependency of dependencies.values()) {
+    dependencyGraph.resolveDependency(
+      parentModule,
+      dependency,
+      transformOptions.platform,
+    );
+  }
+}
+
+function flatten<T>(input: Iterable<Iterable<T>>): Set<T> {
+  const output = new Set();
+
+  for (const items of input) {
+    for (const item of items) {
+      output.add(item);
+    }
+  }
+
+  return output;
+}
+
+module.exports = {
+  initialTraverseDependencies,
+  traverseDependencies,
+};

packages/metro-bundler/src/node-haste/DependencyGraph.js

@@ -237,6 +237,22 @@ class DependencyGraph extends EventEmitter {
     return Promise.resolve(this._moduleCache.getAllModules());
   }
 
+  resolveDependency(
+    fromModule: Module,
+    toModuleName: string,
+    platform: ?string,
+  ): Module {
+    const req = new ResolutionRequest({
+      moduleResolver: this._moduleResolver,
+      entryPath: fromModule.path,
+      helpers: this._helpers,
+      platform: platform || null,
+      moduleCache: this._moduleCache,
+    });
+
+    return req.resolveDependency(fromModule, toModuleName);
+  }
+
   getDependencies<T: {+transformer: JSTransformerOptions}>({
     entryPath,
     options,