module graph

Summary:
This piece of code can assemble all transitive dependencies of a set of entry modules. It is supposed to replace `ResolutionRequest.getOrderedDependencies`.

It has the following advantages:
- has minimal API surface to other components of the system (uses two functions, exposes one function)
- allows to separate concerns into loading + transforming files, resolving dependencies, gathering all modules belonging to a bundle (this code), and bundling
- allows to specify multiple entry points
- allows to use any kind of dependency ID as entry point (haste IDs, node module IDs, relative paths, absolute paths – depends on the resolver)
- allows to skip files, which allows callers to incrementally update previously retrieved collections of modules

Reviewed By: cpojer

Differential Revision: D3627346

fbshipit-source-id: 84b7aa693ca6e89ba3c1ab2af9a004e2e0aaed3d

flow/console.js

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2013-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+declare module 'console' {
+  declare function assert(value: any, ...message: any): void;
+  declare function dir(
+    obj: Object,
+    options: {showHidden: boolean, depth: number, colors: boolean},
+  ): void;
+  declare function error(...data: any): void;
+  declare function info(...data: any): void;
+  declare function log(...data: any): void;
+  declare function time(label: any): void;
+  declare function timeEnd(label: any): void;
+  declare function trace(first: any, ...rest: any): void;
+  declare function warn(...data: any): void;
+  declare class Console {
+    assert(value: any, ...message: any): void,
+    dir(
+      obj: Object,
+      options: {showHidden: boolean, depth: number, colors: boolean},
+    ): void,
+    error(...data: any): void,
+    info(...data: any): void,
+    log(...data: any): void,
+    time(label: any): void,
+    timeEnd(label: any): void,
+    trace(first: any, ...rest: any): void,
+    warn(...data: any): void,
+  }
+}

package.json

@@ -96,7 +96,8 @@
       "source-map",
       "fastpath",
       "denodeify",
-      "fbjs"
+      "fbjs",
+      "sinon"
     ]
   },
   "main": "Libraries/react-native/react-native.js",
@@ -135,6 +136,7 @@
   "dependencies": {
     "absolute-path": "^0.0.0",
     "art": "^0.10.0",
+    "async": "^2.0.1",
     "babel-core": "^6.10.4",
     "babel-plugin-external-helpers": "^6.8.0",
     "babel-plugin-syntax-trailing-function-commas": "^6.5.0",
@@ -215,6 +217,7 @@
     "mock-fs": "^3.11.0",
     "portfinder": "0.4.0",
     "react": "15.3.1-rc.2",
-    "shelljs": "0.6.0"
+    "shelljs": "0.6.0",
+    "sinon": "^2.0.0-pre.2"
   }
 }

packager/react-packager/src/ModuleGraph/Graph.js

@@ -0,0 +1,106 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+const memoize = require('async/memoize');
+const queue = require('async/queue');
+const seq = require('async/seq');
+const invariant = require('fbjs/lib/invariant');
+
+import type {GraphFn, LoadFn, ResolveFn, File, Module} from './types.flow';
+
+const createParentModule =
+  () => ({file: {path: '', ast: {}}, dependencies: []});
+
+const noop = () => {};
+
+exports.create = function create(resolve: ResolveFn, load: LoadFn): GraphFn {
+  function Graph(entryPoints, platform, options = {}, callback = noop) {
+    const {cwd = '', log = (console: any), optimize = false, skip} = options;
+
+    if (typeof platform !== 'string') {
+      log.error('`Graph`, called without a platform');
+      return callback(Error('The target platform has to be passed'));
+    }
+
+    const modules: Map<string | null, Module> = new Map();
+    modules.set(null, createParentModule());
+
+    const loadQueue = queue(seq(
+      ({id, parent}, cb) => resolve(id, parent, platform, options, cb),
+      memoize((file, cb) => load(file, {log, optimize}, cb)),
+    ), Number.MAX_SAFE_INTEGER);
+
+    const cleanup = () => (loadQueue.drain = noop);
+    loadQueue.drain = () => {
+      cleanup();
+      callback(null, collect(null, modules));
+    };
+
+    function loadModule(id: string, parent: string | null, parentDependencyIndex) {
+      function onFileLoaded(
+        error: ?Error,
+        file: File,
+        dependencyIDs: Array<string>,
+      ) {
+        if (error) {
+          cleanup();
+          callback(error);
+          return;
+        }
+
+        const parentModule = modules.get(parent);
+        invariant(parentModule, 'Invalid parent module: ' + String(parent));
+        parentModule.dependencies[parentDependencyIndex] = {id, path: file.path};
+
+        if ((!skip || !skip.has(file.path)) && !modules.has(file.path)) {
+          const dependencies = Array(dependencyIDs.length);
+          modules.set(file.path, {file, dependencies});
+          dependencyIDs.forEach(
+            (dependencyID, j) => loadModule(dependencyID, file.path, j));
+        }
+      }
+      loadQueue.push({id, parent: parent != null ? parent : cwd}, onFileLoaded);
+    }
+
+    let i = 0;
+    for (const entryPoint of entryPoints) {
+      loadModule(entryPoint, null, i++);
+    }
+
+    if (loadQueue.idle()) {
+      log.error('`Graph` called without any entry points');
+      cleanup();
+      callback(Error('At least one entry point has to be passed.'));
+    }
+  }
+
+  return Graph;
+};
+
+function collect(
+  path,
+  modules,
+  serialized = [],
+  seen: Set<string | null> = new Set(),
+): Array<Module> {
+  const module = modules.get(path);
+  if (!module || seen.has(path)) { return serialized; }
+
+  if (path !== null) {
+    serialized.push(module);
+    seen.add(path);
+  }
+  module.dependencies.forEach(
+    dependency => collect(dependency.path, modules, serialized, seen));
+
+  return serialized;
+}

packager/react-packager/src/ModuleGraph/__tests__/Graph-test.js

@@ -0,0 +1,315 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+jest
+  .disableAutomock()
+  .mock('console');
+
+const {Console} = require('console');
+const Graph = require('../Graph');
+const {fn} = require('../test-helpers');
+
+const {any, objectContaining} = jasmine;
+const quiet = new Console();
+
+describe('Graph:', () => {
+  const anyEntry = ['arbitrary/entry/point'];
+  const anyPlatform = 'arbitrary platform';
+  const noOpts = undefined;
+
+  let graph, load, resolve;
+  beforeEach(() => {
+    load = fn();
+    resolve = fn();
+    resolve.stub.yields(null, 'arbitrary file');
+    load.stub.yields(null, createFile('arbitrary file'), []);
+
+    graph = Graph.create(resolve, load);
+  });
+
+  it('calls back an error when called without any entry point', done => {
+    graph([], anyPlatform, {log: quiet}, (error) => {
+      expect(error).toEqual(any(Error));
+      done();
+    });
+  });
+
+  it('resolves the entry point with the passed-in `resolve` function', done => {
+    const entryPoint = '/arbitrary/path';
+    graph([entryPoint], anyPlatform, noOpts, () => {
+      expect(resolve).toBeCalledWith(
+        entryPoint, '', any(String), any(Object), any(Function));
+      done();
+    });
+  });
+
+  it('allows to specify multiple entry points', done => {
+    const entryPoints = ['Arbitrary', '../entry.js'];
+    graph(entryPoints, anyPlatform, noOpts, () => {
+      expect(resolve).toBeCalledWith(
+        entryPoints[0], '', any(String), any(Object), any(Function));
+      expect(resolve).toBeCalledWith(
+        entryPoints[1], '', any(String), any(Object), any(Function));
+      done();
+    });
+
+  });
+
+  it('calls back with an error when called without `platform` option', done => {
+    graph(anyEntry, undefined, {log: quiet}, error => {
+      expect(error).toEqual(any(Error));
+      done();
+    });
+  });
+
+  it('forwards a passed-in `platform` to `resolve`', done => {
+    const platform = 'any';
+    graph(anyEntry, platform, noOpts, () => {
+      expect(resolve).toBeCalledWith(
+        any(String), '', platform, any(Object), any(Function));
+      done();
+    });
+  });
+
+  it('forwards a passed-in `log` option to `resolve`', done => {
+    const log = new Console();
+    graph(anyEntry, anyPlatform, {log}, () => {
+      expect(resolve).toBeCalledWith(
+        any(String), '', any(String), objectContaining({log}), any(Function));
+      done();
+    });
+  });
+
+  it('calls back with every error produced by `resolve`', done => {
+    const error = Error();
+    resolve.stub.yields(error);
+    graph(anyEntry, anyPlatform, noOpts, e => {
+      expect(e).toBe(error);
+      done();
+    });
+  });
+
+  it('passes the files returned by `resolve` on to the `load` function', done => {
+    const modules = new Map([
+      ['Arbitrary', '/absolute/path/to/Arbitrary.js'],
+      ['../entry.js', '/whereever/is/entry.js'],
+    ]);
+    for (const [id, file] of modules) {
+      resolve.stub.withArgs(id).yields(null, file);
+    }
+    const [file1, file2] = modules.values();
+
+    graph(modules.keys(), anyPlatform, noOpts, () => {
+      expect(load).toBeCalledWith(file1, any(Object), any(Function));
+      expect(load).toBeCalledWith(file2, any(Object), any(Function));
+      done();
+    });
+  });
+
+  it('passes the `optimize` flag on to `load`', done => {
+    graph(anyEntry, anyPlatform, {optimize: true}, () => {
+      expect(load).toBeCalledWith(
+        any(String), objectContaining({optimize: true}), any(Function));
+      done();
+    });
+  });
+
+  it('uses `false` as the default for the `optimize` flag', done => {
+    graph(anyEntry, anyPlatform, noOpts, () => {
+      expect(load).toBeCalledWith(
+        any(String), objectContaining({optimize: false}), any(Function));
+      done();
+    });
+  });
+
+  it('forwards a passed-in `log` to `load`', done => {
+    const log = new Console();
+    graph(anyEntry, anyPlatform, {log}, () => {
+      expect(load)
+        .toBeCalledWith(any(String), objectContaining({log}), any(Function));
+      done();
+    });
+  });
+
+  it('calls back with every error produced by `load`', done => {
+    const error = Error();
+    load.stub.yields(error);
+    graph(anyEntry, anyPlatform, noOpts, e => {
+      expect(e).toBe(error);
+      done();
+    });
+  });
+
+  it('resolves any dependencies provided by `load`', done => {
+    const entryPath = '/path/to/entry.js';
+    const id1 = 'required/id';
+    const id2 = './relative/import';
+    resolve.stub.withArgs('entry').yields(null, entryPath);
+    load.stub.withArgs(entryPath)
+      .yields(null, {path: entryPath}, [id1, id2]);
+
+    graph(['entry'], anyPlatform, noOpts, () => {
+      expect(resolve).toBeCalledWith(
+        id1, entryPath, any(String), any(Object), any(Function));
+      expect(resolve).toBeCalledWith(
+        id2, entryPath, any(String), any(Object), any(Function));
+      done();
+    });
+  });
+
+  it('loads transitive dependencies', done => {
+    const entryPath = '/path/to/entry.js';
+    const id1 = 'required/id';
+    const id2 = './relative/import';
+    const path1 = '/path/to/dep/1';
+    const path2 = '/path/to/dep/2';
+
+    resolve.stub
+      .withArgs(id1).yields(null, path1)
+      .withArgs(id2).yields(null, path2)
+      .withArgs('entry').yields(null, entryPath);
+    load.stub
+      .withArgs(entryPath).yields(null, {path: entryPath}, [id1])
+      .withArgs(path1).yields(null, {path: path1}, [id2]);
+
+    graph(['entry'], anyPlatform, noOpts, () => {
+      expect(resolve).toBeCalledWith(id2, path1, any(String), any(Object), any(Function));
+      expect(load).toBeCalledWith(path1, any(Object), any(Function));
+      expect(load).toBeCalledWith(path2, any(Object), any(Function));
+      done();
+    });
+  });
+
+  it('calls back with an array of modules in depth-first traversal order, regardless of the order of resolution', done => {
+    load.stub.reset();
+    resolve.stub.reset();
+
+    const ids = [
+      'a',
+        'b',
+          'c', 'd',
+        'e',
+          'f', 'g',
+        'h',
+    ];
+    ids.forEach(id => {
+      const path = idToPath(id);
+      resolve.stub.withArgs(id).yields(null, path);
+      load.stub.withArgs(path).yields(null, createFile(id), []);
+    });
+    load.stub.withArgs(idToPath('a')).yields(null, createFile('a'), ['b', 'e', 'h']);
+    load.stub.withArgs(idToPath('b')).yields(null, createFile('b'), ['c', 'd']);
+    load.stub.withArgs(idToPath('e')).yields(null, createFile('e'), ['f', 'g']);
+
+    // load certain ids later
+    ['b', 'e', 'h'].forEach(id => resolve.stub.withArgs(id).resetBehavior());
+    resolve.stub.withArgs('h').func = (a, b, c, d, callback) => {
+      callback(null, idToPath('h'));
+      ['e', 'b'].forEach(
+        id => resolve.stub.withArgs(id).yield(null, idToPath(id)));
+    };
+
+    graph(['a'], anyPlatform, noOpts, (error, result) => {
+      expect(error).toEqual(null);
+      expect(result).toEqual([
+        createModule('a', ['b', 'e', 'h']),
+        createModule('b', ['c', 'd']),
+        createModule('c'),
+        createModule('d'),
+        createModule('e', ['f', 'g']),
+        createModule('f'),
+        createModule('g'),
+        createModule('h'),
+      ]);
+      done();
+    });
+  });
+
+  it('does not include dependencies more than once', done => {
+    const ids = ['a', 'b', 'c', 'd'];
+    ids.forEach(id => {
+      const path = idToPath(id);
+      resolve.stub.withArgs(id).yields(null, path);
+      load.stub.withArgs(path).yields(null, createFile(id), []);
+    });
+    ['a', 'd'].forEach(id =>
+      load.stub
+        .withArgs(idToPath(id)).yields(null, createFile(id), ['b', 'c']));
+
+    graph(['a', 'd', 'b'], anyPlatform, noOpts, (error, result) => {
+      expect(error).toEqual(null);
+      expect(result).toEqual([
+        createModule('a', ['b', 'c']),
+        createModule('b'),
+        createModule('c'),
+        createModule('d', ['b', 'c']),
+      ]);
+      done();
+    });
+  });
+
+  it('handles dependency cycles', done => {
+    resolve.stub
+      .withArgs('a').yields(null, idToPath('a'))
+      .withArgs('b').yields(null, idToPath('b'))
+      .withArgs('c').yields(null, idToPath('c'));
+    load.stub
+      .withArgs(idToPath('a')).yields(null, createFile('a'), ['b'])
+      .withArgs(idToPath('b')).yields(null, createFile('b'), ['c'])
+      .withArgs(idToPath('c')).yields(null, createFile('c'), ['a']);
+
+    graph(['a'], anyPlatform, noOpts, (error, result) => {
+      expect(result).toEqual([
+        createModule('a', ['b']),
+        createModule('b', ['c']),
+        createModule('c', ['a']),
+      ]);
+      done();
+    });
+  });
+
+  it('can skip files', done => {
+    ['a', 'b', 'c', 'd', 'e'].forEach(
+      id => resolve.stub.withArgs(id).yields(null, idToPath(id)));
+    load.stub
+      .withArgs(idToPath('a')).yields(null, createFile('a'), ['b', 'c', 'd'])
+      .withArgs(idToPath('b')).yields(null, createFile('b'), ['e']);
+    ['c', 'd', 'e'].forEach(id =>
+      load.stub.withArgs(idToPath(id)).yields(null, createFile(id), []));
+    const skip = new Set([idToPath('b'), idToPath('c')]);
+
+    graph(['a'], anyPlatform, {skip}, (error, result) => {
+      expect(result).toEqual([
+        createModule('a', ['b', 'c', 'd']),
+        createModule('d', []),
+      ]);
+      done();
+    });
+  });
+});
+
+function createDependency(id) {
+  return {id, path: idToPath(id)};
+}
+
+function createFile(id) {
+  return {ast: {}, path: idToPath(id)};
+}
+
+function createModule(id, dependencies = []): Module {
+  return {
+    file: createFile(id),
+    dependencies: dependencies.map(createDependency)
+  };
+}
+
+function idToPath(id) {
+  return '/path/to/' + id;
+}

packager/react-packager/src/ModuleGraph/silent-console.js

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const {Console} = require('console');
+const {Writable} = require('stream');
+
+const write = (_, __, callback) => callback();
+module.exports = new Console(new Writable({write, writev: write}));

packager/react-packager/src/ModuleGraph/test-helpers.js

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const stub = require('sinon/lib/sinon/stub');
+
+exports.fn = () => {
+  const s = stub();
+  const f = jest.fn(s);
+  f.stub = s;
+  return f;
+};

packager/react-packager/src/ModuleGraph/types.flow.js

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+import type {Console} from 'console';
+
+type callback<T> = (error: ?Error, result?: T) => any;
+type callback2<T, T1> = (error: ?Error, a?: T, b?: T1) => any;
+
+type ResolveOptions = {
+  log?: Console,
+};
+
+type LoadOptions = {
+  log?: Console,
+  optimize?: boolean,
+  platform?: string,
+};
+
+type GraphOptions = {
+  cwd?: string,
+  log?: Console,
+  optimize?: boolean,
+  skip?: Set<string>,
+};
+
+type Dependency = {
+  id: string,
+  path: string,
+};
+
+export type File = {
+  path: string,
+  code?: string,
+  ast: Object,
+};
+
+export type Module = {
+  file: File,
+  dependencies: Array<Dependency>,
+};
+
+export type GraphFn = (
+  entryPoints: Iterable<string>,
+  platform: string,
+  options?: GraphOptions,
+  callback?: callback<Array<Module>>,
+) => void;
+
+export type ResolveFn = (
+  id: string,
+  source: string,
+  platform: string,
+  options?: ResolveOptions,
+  callback: callback<string>,
+) => void;
+
+export type LoadFn = (
+  file: string,
+  options: LoadOptions,
+  callback: callback2<File, Array<string>>,
+) => void;