packager: index files by dir for fast matching

Summary: This removes the call to `HasteFS#matchFiles()`, that has linear complexity. Instead, we index all the files by directory from `HasteFS` once loaded, a linear operation. Then, we can filter files from a particular directory much quicker.

Reviewed By: davidaurelio

Differential Revision: D4826721

fbshipit-source-id: c31a0ed9a354dbc7f2dcd56179b859e491faa16c

packager/src/ModuleGraph/node-haste/node-haste.js

@@ -22,6 +22,7 @@ import type {
 } from '../types.flow';
 
 const DependencyGraphHelpers = require('../../node-haste/DependencyGraph/DependencyGraphHelpers');
+const FilesByDirNameIndex = require('../../node-haste/FilesByDirNameIndex');
 const HasteFS = require('./HasteFS');
 const HasteMap = require('../../node-haste/DependencyGraph/HasteMap');
 const Module = require('./Module');
@@ -92,6 +93,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn {
 
   const hasteMapBuilt = hasteMap.build();
   const resolutionRequests = {};
+  const filesByDirNameIndex = new FilesByDirNameIndex(hasteMap.getAllFiles());
   return (id, source, platform, _, callback) => {
     let resolutionRequest = resolutionRequests[platform];
     if (!resolutionRequest) {
@@ -102,6 +104,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn {
         hasteFS,
         hasteMap,
         helpers,
+        matchFiles: filesByDirNameIndex.match.bind(filesByDirNameIndex),
         moduleCache,
         moduleMap: getFakeModuleMap(hasteMap),
         platform,

packager/src/node-haste/DependencyGraph/ResolutionRequest.js

@@ -53,12 +53,15 @@ type ModuleishCache<TModule, TPackage> = {
   getAssetModule(path: string): TModule,
 };
 
+type MatchFilesByDirAndPattern = (dirName: string, pattern: RegExp) => Array<string>;
+
 type Options<TModule, TPackage> = {
   dirExists: DirExistsFn,
   entryPath: string,
   extraNodeModules: ?Object,
   hasteFS: HasteFS,
   helpers: DependencyGraphHelpers,
+  matchFiles: MatchFilesByDirAndPattern,
   moduleCache: ModuleishCache<TModule, TPackage>,
   moduleMap: ModuleMap,
   platform: string,
@@ -89,6 +92,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
   _hasteFS: HasteFS;
   _helpers: DependencyGraphHelpers;
   _immediateResolutionCache: {[key: string]: TModule};
+  _matchFiles: MatchFilesByDirAndPattern;
   _moduleCache: ModuleishCache<TModule, TPackage>;
   _moduleMap: ModuleMap;
   _platform: string;
@@ -102,6 +106,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
     extraNodeModules,
     hasteFS,
     helpers,
+    matchFiles,
     moduleCache,
     moduleMap,
     platform,
@@ -113,6 +118,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
     this._extraNodeModules = extraNodeModules;
     this._hasteFS = hasteFS;
     this._helpers = helpers;
+    this._matchFiles = matchFiles;
     this._moduleCache = moduleCache;
     this._moduleMap = moduleMap;
     this._platform = platform;
@@ -442,7 +448,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
 
   _loadAsFile(potentialModulePath: string, fromModule: TModule, toModule: string): TModule {
     if (this._helpers.isAssetFile(potentialModulePath)) {
-      let dirname = path.dirname(potentialModulePath);
+      const dirname = path.dirname(potentialModulePath);
       if (!this._dirExists(dirname)) {
         throw new UnableToResolveError(
           fromModule,
@@ -453,22 +459,16 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
 
       const {name, type} = getAssetDataFromName(potentialModulePath, this._platforms);
 
-      let pattern = name + '(@[\\d\\.]+x)?';
+      let pattern = '^' + name + '(@[\\d\\.]+x)?';
       if (this._platform != null) {
         pattern += '(\\.' + this._platform + ')?';
       }
-      pattern += '\\.' + type;
+      pattern += '\\.' + type + '$';
 
-      // Escape backslashes in the path to be able to use it in the regex
-      if (path.sep === '\\') {
-        dirname = dirname.replace(/\\/g, '\\\\');
-      }
-
-      // We arbitrarly grab the first one, because scale selection
-      // will happen somewhere
-      const [assetFile] = this._hasteFS.matchFiles(
-        new RegExp(dirname + '(\/|\\\\)' + pattern)
-      );
+      const assetFiles = this._matchFiles(dirname, new RegExp(pattern));
+      // We arbitrarly grab the lowest, because scale selection will happen
+      // somewhere else. Always the lowest so that it's stable between builds.
+      const assetFile = getArrayLowestItem(assetFiles);
       if (assetFile) {
         return this._moduleCache.getAssetModule(assetFile);
       }
@@ -582,6 +582,19 @@ function isRelativeImport(filePath) {
   return /^[.][.]?(?:[/]|$)/.test(filePath);
 }
 
+function getArrayLowestItem(a: Array<string>): string | void {
+  if (a.length === 0) {
+    return undefined;
+  }
+  let lowest = a[0];
+  for (let i = 1; i < a.length; ++i) {
+    if (a[i] < lowest) {
+      lowest = a[i];
+    }
+  }
+  return lowest;
+}
+
 ResolutionRequest.emptyModule = require.resolve('./assets/empty-module.js');
 
 module.exports = ResolutionRequest;

packager/src/node-haste/FilesByDirNameIndex.js

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+const path = require('path');
+
+/**
+ * This is a way to find files quickly given a RegExp, in a specific directory.
+ * This is must faster than iterating over all the files and matching both
+ * directory and RegExp at the same time.
+ *
+ * This was first implemented to support finding assets fast, for which we know
+ * the directory, but we want to identify all variants (ex. @2x, @1x, for
+ * a picture's different definition levels).
+ */
+class FilesByDirNameIndex {
+  _filesByDirName: Map<string, Array<string>>;
+
+  constructor(allFilePaths: Array<string>) {
+    this._filesByDirName = new Map();
+    for (let i = 0; i < allFilePaths.length; ++i) {
+      const filePath = allFilePaths[i];
+      const dirName = path.dirname(filePath);
+      let dir = this._filesByDirName.get(dirName);
+      if (dir === undefined) {
+        dir = [];
+        this._filesByDirName.set(dirName, dir);
+      }
+      dir.push(path.basename(filePath));
+    }
+  }
+
+  match(dirName: string, pattern: RegExp): Array<string> {
+    const results = [];
+    const dir = this._filesByDirName.get(dirName);
+    if (dir === undefined) {
+      return [];
+    }
+    for (let i = 0; i < dir.length; ++i) {
+      if (pattern.test(dir[i])) {
+        results.push(path.join(dirName, dir[i]));
+      }
+    }
+    return results;
+  }
+}
+
+module.exports = FilesByDirNameIndex;

packager/src/node-haste/index.js

@@ -13,6 +13,7 @@
 
 const Cache = require('./Cache');
 const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers');
+const FilesByDirNameIndex = require('./FilesByDirNameIndex');
 const JestHasteMap = require('jest-haste-map');
 const Module = require('./Module');
 const ModuleCache = require('./ModuleCache');
@@ -76,6 +77,7 @@ const JEST_HASTE_MAP_CACHE_BREAKER = 1;
 class DependencyGraph extends EventEmitter {
 
   _opts: Options;
+  _filesByDirNameIndex: FilesByDirNameIndex;
   _haste: JestHasteMap;
   _helpers: DependencyGraphHelpers;
   _moduleCache: ModuleCache;
@@ -91,12 +93,14 @@ class DependencyGraph extends EventEmitter {
     super();
     invariant(config.opts.maxWorkerCount >= 1, 'worker count must be greater or equal to 1');
     this._opts = config.opts;
+    this._filesByDirNameIndex = new FilesByDirNameIndex(config.initialHasteFS.getAllFiles());
     this._haste = config.haste;
     this._hasteFS = config.initialHasteFS;
     this._moduleMap = config.initialModuleMap;
     this._helpers = new DependencyGraphHelpers(this._opts);
     this._haste.on('change', this._onHasteChange.bind(this));
     this._moduleCache = this._createModuleCache();
+    (this: any)._matchFilesByDirAndPattern = this._matchFilesByDirAndPattern.bind(this);
   }
 
   static _createHaste(opts: Options): JestHasteMap {
@@ -149,6 +153,7 @@ class DependencyGraph extends EventEmitter {
 
   _onHasteChange({eventsQueue, hasteFS, moduleMap}) {
     this._hasteFS = hasteFS;
+    this._filesByDirNameIndex = new FilesByDirNameIndex(hasteFS.getAllFiles());
     this._moduleMap = moduleMap;
     eventsQueue.forEach(({type, filePath, stat}) =>
       this._moduleCache.processFileChange(type, filePath, stat)
@@ -226,6 +231,7 @@ class DependencyGraph extends EventEmitter {
       extraNodeModules: this._opts.extraNodeModules,
       hasteFS: this._hasteFS,
       helpers: this._helpers,
+      matchFiles: this._matchFilesByDirAndPattern,
       moduleCache: this._moduleCache,
       moduleMap: this._moduleMap,
       platform,
@@ -243,6 +249,10 @@ class DependencyGraph extends EventEmitter {
     }).then(() => response);
   }
 
+  _matchFilesByDirAndPattern(dirName: string, pattern: RegExp) {
+    return this._filesByDirNameIndex.match(dirName, pattern);
+  }
+
   matchFilesByPattern(pattern: RegExp) {
     return Promise.resolve(this._hasteFS.matchFiles(pattern));
   }