From 8d34f16a70983e789ecec9fd09f5019325ea7fc6 Mon Sep 17 00:00:00 2001 From: Jean Lauliac Date: Thu, 6 Apr 2017 07:45:36 -0700 Subject: [PATCH] packager: index files by dir for fast matching Summary: This removes the call to `HasteFS#matchFiles()`, that has linear complexity. Instead, we index all the files by directory from `HasteFS` once loaded, a linear operation. Then, we can filter files from a particular directory much quicker. Reviewed By: davidaurelio Differential Revision: D4826721 fbshipit-source-id: c31a0ed9a354dbc7f2dcd56179b859e491faa16c --- .../src/ModuleGraph/node-haste/node-haste.js | 3 + .../DependencyGraph/ResolutionRequest.js | 39 ++++++++----- .../src/node-haste/FilesByDirNameIndex.js | 57 +++++++++++++++++++ .../metro-bundler/src/node-haste/index.js | 10 ++++ 4 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 packages/metro-bundler/src/node-haste/FilesByDirNameIndex.js diff --git a/packages/metro-bundler/src/ModuleGraph/node-haste/node-haste.js b/packages/metro-bundler/src/ModuleGraph/node-haste/node-haste.js index e3cbd224..0d643e3e 100644 --- a/packages/metro-bundler/src/ModuleGraph/node-haste/node-haste.js +++ b/packages/metro-bundler/src/ModuleGraph/node-haste/node-haste.js @@ -22,6 +22,7 @@ import type { } from '../types.flow'; const DependencyGraphHelpers = require('../../node-haste/DependencyGraph/DependencyGraphHelpers'); +const FilesByDirNameIndex = require('../../node-haste/FilesByDirNameIndex'); const HasteFS = require('./HasteFS'); const HasteMap = require('../../node-haste/DependencyGraph/HasteMap'); const Module = require('./Module'); @@ -92,6 +93,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn { const hasteMapBuilt = hasteMap.build(); const resolutionRequests = {}; + const filesByDirNameIndex = new FilesByDirNameIndex(hasteMap.getAllFiles()); return (id, source, platform, _, callback) => { let resolutionRequest = resolutionRequests[platform]; if (!resolutionRequest) { @@ -102,6 +104,7 @@ exports.createResolveFn = function(options: ResolveOptions): ResolveFn { hasteFS, hasteMap, helpers, + matchFiles: filesByDirNameIndex.match.bind(filesByDirNameIndex), moduleCache, moduleMap: getFakeModuleMap(hasteMap), platform, diff --git a/packages/metro-bundler/src/node-haste/DependencyGraph/ResolutionRequest.js b/packages/metro-bundler/src/node-haste/DependencyGraph/ResolutionRequest.js index b4336c93..77ec18e0 100644 --- a/packages/metro-bundler/src/node-haste/DependencyGraph/ResolutionRequest.js +++ b/packages/metro-bundler/src/node-haste/DependencyGraph/ResolutionRequest.js @@ -53,12 +53,15 @@ type ModuleishCache = { getAssetModule(path: string): TModule, }; +type MatchFilesByDirAndPattern = (dirName: string, pattern: RegExp) => Array; + type Options = { dirExists: DirExistsFn, entryPath: string, extraNodeModules: ?Object, hasteFS: HasteFS, helpers: DependencyGraphHelpers, + matchFiles: MatchFilesByDirAndPattern, moduleCache: ModuleishCache, moduleMap: ModuleMap, platform: string, @@ -89,6 +92,7 @@ class ResolutionRequest { _hasteFS: HasteFS; _helpers: DependencyGraphHelpers; _immediateResolutionCache: {[key: string]: TModule}; + _matchFiles: MatchFilesByDirAndPattern; _moduleCache: ModuleishCache; _moduleMap: ModuleMap; _platform: string; @@ -102,6 +106,7 @@ class ResolutionRequest { extraNodeModules, hasteFS, helpers, + matchFiles, moduleCache, moduleMap, platform, @@ -113,6 +118,7 @@ class ResolutionRequest { this._extraNodeModules = extraNodeModules; this._hasteFS = hasteFS; this._helpers = helpers; + this._matchFiles = matchFiles; this._moduleCache = moduleCache; this._moduleMap = moduleMap; this._platform = platform; @@ -442,7 +448,7 @@ class ResolutionRequest { _loadAsFile(potentialModulePath: string, fromModule: TModule, toModule: string): TModule { if (this._helpers.isAssetFile(potentialModulePath)) { - let dirname = path.dirname(potentialModulePath); + const dirname = path.dirname(potentialModulePath); if (!this._dirExists(dirname)) { throw new UnableToResolveError( fromModule, @@ -453,22 +459,16 @@ class ResolutionRequest { const {name, type} = getAssetDataFromName(potentialModulePath, this._platforms); - let pattern = name + '(@[\\d\\.]+x)?'; + let pattern = '^' + name + '(@[\\d\\.]+x)?'; if (this._platform != null) { pattern += '(\\.' + this._platform + ')?'; } - pattern += '\\.' + type; + pattern += '\\.' + type + '$'; - // Escape backslashes in the path to be able to use it in the regex - if (path.sep === '\\') { - dirname = dirname.replace(/\\/g, '\\\\'); - } - - // We arbitrarly grab the first one, because scale selection - // will happen somewhere - const [assetFile] = this._hasteFS.matchFiles( - new RegExp(dirname + '(\/|\\\\)' + pattern) - ); + const assetFiles = this._matchFiles(dirname, new RegExp(pattern)); + // We arbitrarly grab the lowest, because scale selection will happen + // somewhere else. Always the lowest so that it's stable between builds. + const assetFile = getArrayLowestItem(assetFiles); if (assetFile) { return this._moduleCache.getAssetModule(assetFile); } @@ -582,6 +582,19 @@ function isRelativeImport(filePath) { return /^[.][.]?(?:[/]|$)/.test(filePath); } +function getArrayLowestItem(a: Array): string | void { + if (a.length === 0) { + return undefined; + } + let lowest = a[0]; + for (let i = 1; i < a.length; ++i) { + if (a[i] < lowest) { + lowest = a[i]; + } + } + return lowest; +} + ResolutionRequest.emptyModule = require.resolve('./assets/empty-module.js'); module.exports = ResolutionRequest; diff --git a/packages/metro-bundler/src/node-haste/FilesByDirNameIndex.js b/packages/metro-bundler/src/node-haste/FilesByDirNameIndex.js new file mode 100644 index 00000000..0da11bf9 --- /dev/null +++ b/packages/metro-bundler/src/node-haste/FilesByDirNameIndex.js @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ + +'use strict'; + +const path = require('path'); + +/** + * This is a way to find files quickly given a RegExp, in a specific directory. + * This is must faster than iterating over all the files and matching both + * directory and RegExp at the same time. + * + * This was first implemented to support finding assets fast, for which we know + * the directory, but we want to identify all variants (ex. @2x, @1x, for + * a picture's different definition levels). + */ +class FilesByDirNameIndex { + _filesByDirName: Map>; + + constructor(allFilePaths: Array) { + this._filesByDirName = new Map(); + for (let i = 0; i < allFilePaths.length; ++i) { + const filePath = allFilePaths[i]; + const dirName = path.dirname(filePath); + let dir = this._filesByDirName.get(dirName); + if (dir === undefined) { + dir = []; + this._filesByDirName.set(dirName, dir); + } + dir.push(path.basename(filePath)); + } + } + + match(dirName: string, pattern: RegExp): Array { + const results = []; + const dir = this._filesByDirName.get(dirName); + if (dir === undefined) { + return []; + } + for (let i = 0; i < dir.length; ++i) { + if (pattern.test(dir[i])) { + results.push(path.join(dirName, dir[i])); + } + } + return results; + } +} + +module.exports = FilesByDirNameIndex; diff --git a/packages/metro-bundler/src/node-haste/index.js b/packages/metro-bundler/src/node-haste/index.js index 6f521fbf..66ce8452 100644 --- a/packages/metro-bundler/src/node-haste/index.js +++ b/packages/metro-bundler/src/node-haste/index.js @@ -13,6 +13,7 @@ const Cache = require('./Cache'); const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers'); +const FilesByDirNameIndex = require('./FilesByDirNameIndex'); const JestHasteMap = require('jest-haste-map'); const Module = require('./Module'); const ModuleCache = require('./ModuleCache'); @@ -76,6 +77,7 @@ const JEST_HASTE_MAP_CACHE_BREAKER = 1; class DependencyGraph extends EventEmitter { _opts: Options; + _filesByDirNameIndex: FilesByDirNameIndex; _haste: JestHasteMap; _helpers: DependencyGraphHelpers; _moduleCache: ModuleCache; @@ -91,12 +93,14 @@ class DependencyGraph extends EventEmitter { super(); invariant(config.opts.maxWorkerCount >= 1, 'worker count must be greater or equal to 1'); this._opts = config.opts; + this._filesByDirNameIndex = new FilesByDirNameIndex(config.initialHasteFS.getAllFiles()); this._haste = config.haste; this._hasteFS = config.initialHasteFS; this._moduleMap = config.initialModuleMap; this._helpers = new DependencyGraphHelpers(this._opts); this._haste.on('change', this._onHasteChange.bind(this)); this._moduleCache = this._createModuleCache(); + (this: any)._matchFilesByDirAndPattern = this._matchFilesByDirAndPattern.bind(this); } static _createHaste(opts: Options): JestHasteMap { @@ -149,6 +153,7 @@ class DependencyGraph extends EventEmitter { _onHasteChange({eventsQueue, hasteFS, moduleMap}) { this._hasteFS = hasteFS; + this._filesByDirNameIndex = new FilesByDirNameIndex(hasteFS.getAllFiles()); this._moduleMap = moduleMap; eventsQueue.forEach(({type, filePath, stat}) => this._moduleCache.processFileChange(type, filePath, stat) @@ -226,6 +231,7 @@ class DependencyGraph extends EventEmitter { extraNodeModules: this._opts.extraNodeModules, hasteFS: this._hasteFS, helpers: this._helpers, + matchFiles: this._matchFilesByDirAndPattern, moduleCache: this._moduleCache, moduleMap: this._moduleMap, platform, @@ -243,6 +249,10 @@ class DependencyGraph extends EventEmitter { }).then(() => response); } + _matchFilesByDirAndPattern(dirName: string, pattern: RegExp) { + return this._filesByDirNameIndex.match(dirName, pattern); + } + matchFilesByPattern(pattern: RegExp) { return Promise.resolve(this._hasteFS.matchFiles(pattern)); }