packager: index files by dir for fast matching

Summary: This removes the call to `HasteFS#matchFiles()`, that has linear complexity. Instead, we index all the files by directory from `HasteFS` once loaded, a linear operation. Then, we can filter files from a particular directory much quicker.

Reviewed By: davidaurelio

Differential Revision: D4826721

fbshipit-source-id: c31a0ed9a354dbc7f2dcd56179b859e491faa16c
This commit is contained in:
Jean Lauliac 2017-04-06 07:45:36 -07:00 committed by Facebook Github Bot
parent 5f37483466
commit b2101836dc
4 changed files with 96 additions and 13 deletions

View File

@ -22,6 +22,7 @@ import type {
} from '../types.flow';
const DependencyGraphHelpers = require('../../node-haste/DependencyGraph/DependencyGraphHelpers');
const FilesByDirNameIndex = require('../../node-haste/FilesByDirNameIndex');
const HasteFS = require('./HasteFS');
const HasteMap = require('../../node-haste/DependencyGraph/HasteMap');
const Module = require('./Module');
@ -92,6 +93,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn {
const hasteMapBuilt = hasteMap.build();
const resolutionRequests = {};
const filesByDirNameIndex = new FilesByDirNameIndex(hasteMap.getAllFiles());
return (id, source, platform, _, callback) => {
let resolutionRequest = resolutionRequests[platform];
if (!resolutionRequest) {
@ -102,6 +104,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn {
hasteFS,
hasteMap,
helpers,
matchFiles: filesByDirNameIndex.match.bind(filesByDirNameIndex),
moduleCache,
moduleMap: getFakeModuleMap(hasteMap),
platform,

View File

@ -53,12 +53,15 @@ type ModuleishCache<TModule, TPackage> = {
getAssetModule(path: string): TModule,
};
type MatchFilesByDirAndPattern = (dirName: string, pattern: RegExp) => Array<string>;
type Options<TModule, TPackage> = {
dirExists: DirExistsFn,
entryPath: string,
extraNodeModules: ?Object,
hasteFS: HasteFS,
helpers: DependencyGraphHelpers,
matchFiles: MatchFilesByDirAndPattern,
moduleCache: ModuleishCache<TModule, TPackage>,
moduleMap: ModuleMap,
platform: string,
@ -89,6 +92,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
_hasteFS: HasteFS;
_helpers: DependencyGraphHelpers;
_immediateResolutionCache: {[key: string]: TModule};
_matchFiles: MatchFilesByDirAndPattern;
_moduleCache: ModuleishCache<TModule, TPackage>;
_moduleMap: ModuleMap;
_platform: string;
@ -102,6 +106,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
extraNodeModules,
hasteFS,
helpers,
matchFiles,
moduleCache,
moduleMap,
platform,
@ -113,6 +118,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
this._extraNodeModules = extraNodeModules;
this._hasteFS = hasteFS;
this._helpers = helpers;
this._matchFiles = matchFiles;
this._moduleCache = moduleCache;
this._moduleMap = moduleMap;
this._platform = platform;
@ -442,7 +448,7 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
_loadAsFile(potentialModulePath: string, fromModule: TModule, toModule: string): TModule {
if (this._helpers.isAssetFile(potentialModulePath)) {
let dirname = path.dirname(potentialModulePath);
const dirname = path.dirname(potentialModulePath);
if (!this._dirExists(dirname)) {
throw new UnableToResolveError(
fromModule,
@ -453,22 +459,16 @@ class ResolutionRequest<TModule: Moduleish, TPackage: Packageish> {
const {name, type} = getAssetDataFromName(potentialModulePath, this._platforms);
let pattern = name + '(@[\\d\\.]+x)?';
let pattern = '^' + name + '(@[\\d\\.]+x)?';
if (this._platform != null) {
pattern += '(\\.' + this._platform + ')?';
}
pattern += '\\.' + type;
pattern += '\\.' + type + '$';
// Escape backslashes in the path to be able to use it in the regex
if (path.sep === '\\') {
dirname = dirname.replace(/\\/g, '\\\\');
}
// We arbitrarly grab the first one, because scale selection
// will happen somewhere
const [assetFile] = this._hasteFS.matchFiles(
new RegExp(dirname + '(\/|\\\\)' + pattern)
);
const assetFiles = this._matchFiles(dirname, new RegExp(pattern));
// We arbitrarly grab the lowest, because scale selection will happen
// somewhere else. Always the lowest so that it's stable between builds.
const assetFile = getArrayLowestItem(assetFiles);
if (assetFile) {
return this._moduleCache.getAssetModule(assetFile);
}
@ -582,6 +582,19 @@ function isRelativeImport(filePath) {
return /^[.][.]?(?:[/]|$)/.test(filePath);
}
function getArrayLowestItem(a: Array<string>): string | void {
if (a.length === 0) {
return undefined;
}
let lowest = a[0];
for (let i = 1; i < a.length; ++i) {
if (a[i] < lowest) {
lowest = a[i];
}
}
return lowest;
}
ResolutionRequest.emptyModule = require.resolve('./assets/empty-module.js');
module.exports = ResolutionRequest;

View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const path = require('path');
/**
* This is a way to find files quickly given a RegExp, in a specific directory.
* This is must faster than iterating over all the files and matching both
* directory and RegExp at the same time.
*
* This was first implemented to support finding assets fast, for which we know
* the directory, but we want to identify all variants (ex. @2x, @1x, for
* a picture's different definition levels).
*/
class FilesByDirNameIndex {
_filesByDirName: Map<string, Array<string>>;
constructor(allFilePaths: Array<string>) {
this._filesByDirName = new Map();
for (let i = 0; i < allFilePaths.length; ++i) {
const filePath = allFilePaths[i];
const dirName = path.dirname(filePath);
let dir = this._filesByDirName.get(dirName);
if (dir === undefined) {
dir = [];
this._filesByDirName.set(dirName, dir);
}
dir.push(path.basename(filePath));
}
}
match(dirName: string, pattern: RegExp): Array<string> {
const results = [];
const dir = this._filesByDirName.get(dirName);
if (dir === undefined) {
return [];
}
for (let i = 0; i < dir.length; ++i) {
if (pattern.test(dir[i])) {
results.push(path.join(dirName, dir[i]));
}
}
return results;
}
}
module.exports = FilesByDirNameIndex;

View File

@ -13,6 +13,7 @@
const Cache = require('./Cache');
const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers');
const FilesByDirNameIndex = require('./FilesByDirNameIndex');
const JestHasteMap = require('jest-haste-map');
const Module = require('./Module');
const ModuleCache = require('./ModuleCache');
@ -76,6 +77,7 @@ const JEST_HASTE_MAP_CACHE_BREAKER = 1;
class DependencyGraph extends EventEmitter {
_opts: Options;
_filesByDirNameIndex: FilesByDirNameIndex;
_haste: JestHasteMap;
_helpers: DependencyGraphHelpers;
_moduleCache: ModuleCache;
@ -91,12 +93,14 @@ class DependencyGraph extends EventEmitter {
super();
invariant(config.opts.maxWorkerCount >= 1, 'worker count must be greater or equal to 1');
this._opts = config.opts;
this._filesByDirNameIndex = new FilesByDirNameIndex(config.initialHasteFS.getAllFiles());
this._haste = config.haste;
this._hasteFS = config.initialHasteFS;
this._moduleMap = config.initialModuleMap;
this._helpers = new DependencyGraphHelpers(this._opts);
this._haste.on('change', this._onHasteChange.bind(this));
this._moduleCache = this._createModuleCache();
(this: any)._matchFilesByDirAndPattern = this._matchFilesByDirAndPattern.bind(this);
}
static _createHaste(opts: Options): JestHasteMap {
@ -149,6 +153,7 @@ class DependencyGraph extends EventEmitter {
_onHasteChange({eventsQueue, hasteFS, moduleMap}) {
this._hasteFS = hasteFS;
this._filesByDirNameIndex = new FilesByDirNameIndex(hasteFS.getAllFiles());
this._moduleMap = moduleMap;
eventsQueue.forEach(({type, filePath, stat}) =>
this._moduleCache.processFileChange(type, filePath, stat)
@ -226,6 +231,7 @@ class DependencyGraph extends EventEmitter {
extraNodeModules: this._opts.extraNodeModules,
hasteFS: this._hasteFS,
helpers: this._helpers,
matchFiles: this._matchFilesByDirAndPattern,
moduleCache: this._moduleCache,
moduleMap: this._moduleMap,
platform,
@ -243,6 +249,10 @@ class DependencyGraph extends EventEmitter {
}).then(() => response);
}
_matchFilesByDirAndPattern(dirName: string, pattern: RegExp) {
return this._filesByDirNameIndex.match(dirName, pattern);
}
matchFilesByPattern(pattern: RegExp) {
return Promise.resolve(this._hasteFS.matchFiles(pattern));
}