From ecd933e73d8104fb5fac62687cf0bb4eb3817873 Mon Sep 17 00:00:00 2001 From: Ovidiu Viorel Iepure Date: Mon, 3 Oct 2016 05:16:04 -0700 Subject: [PATCH] Replacing node-haste with jest-haste-map Summary: Modified `node-haste` implementation to use the much faster `jest-haste-map` under the hood. The underlying `fastfs` now gets passed the entire file list from the `jest-haste-map` rather than crawl the filesystem. Reviewed By: cpojer Differential Revision: D3724387 fbshipit-source-id: 447d58ea0edf283662ec23d1e2deee992cf8d240 --- react-packager/src/Bundler/index.js | 17 +- react-packager/src/Resolver/index.js | 6 + .../src/Server/__tests__/Server-test.js | 1 + react-packager/src/Server/index.js | 18 +- .../DependencyGraph/DeprecatedAssetMap.js | 57 +--- .../src/node-haste/__mocks__/graceful-fs.js | 34 ++ .../__tests__/DependencyGraph-test.js | 292 ++++++++++++------ .../src/node-haste/__tests__/Module-test.js | 7 +- .../__tests__/fastfs-integrated-test.js | 12 +- .../src/node-haste/crawlers/index.js | 13 - .../src/node-haste/crawlers/node.js | 61 ---- .../src/node-haste/crawlers/watchman.js | 76 ----- react-packager/src/node-haste/fastfs.js | 64 ++-- react-packager/src/node-haste/index.js | 230 ++++++++------ 14 files changed, 435 insertions(+), 453 deletions(-) delete mode 100644 react-packager/src/node-haste/crawlers/index.js delete mode 100644 react-packager/src/node-haste/crawlers/node.js delete mode 100644 react-packager/src/node-haste/crawlers/watchman.js diff --git a/react-packager/src/Bundler/index.js b/react-packager/src/Bundler/index.js index 8bf16e6b..e9ea7352 100644 --- a/react-packager/src/Bundler/index.js +++ b/react-packager/src/Bundler/index.js @@ -143,19 +143,20 @@ class Bundler { }); this._resolver = new Resolver({ - projectRoots: opts.projectRoots, - blacklistRE: opts.blacklistRE, - polyfillModuleNames: opts.polyfillModuleNames, - moduleFormat: opts.moduleFormat, - assetRoots: opts.assetRoots, - fileWatcher: opts.fileWatcher, assetExts: opts.assetExts, + assetRoots: opts.assetRoots, + blacklistRE: opts.blacklistRE, cache: this._cache, + extraNodeModules: opts.extraNodeModules, + fileWatcher: opts.fileWatcher, + minifyCode: this._transformer.minify, + moduleFormat: opts.moduleFormat, + polyfillModuleNames: opts.polyfillModuleNames, + projectRoots: opts.projectRoots, + resetCache: opts.resetCache, transformCode: (module, code, options) => this._transformer.transformFile(module.path, code, options), - extraNodeModules: opts.extraNodeModules, - minifyCode: this._transformer.minify, }); this._projectRoots = opts.projectRoots; diff --git a/react-packager/src/Resolver/index.js b/react-packager/src/Resolver/index.js index 273e7142..cefecd59 100644 --- a/react-packager/src/Resolver/index.js +++ b/react-packager/src/Resolver/index.js @@ -57,6 +57,10 @@ const validateOpts = declareOpts({ minifyCode: { type: 'function', }, + resetCache: { + type: 'boolean', + default: false, + }, }); const getDependenciesValidateOpts = declareOpts({ @@ -109,6 +113,8 @@ class Resolver { transformCode: opts.transformCode, extraNodeModules: opts.extraNodeModules, assetDependencies: ['react-native/Libraries/Image/AssetRegistry'], + // for jest-haste-map + resetCache: options.resetCache, }); this._minifyCode = opts.minifyCode; diff --git a/react-packager/src/Server/__tests__/Server-test.js b/react-packager/src/Server/__tests__/Server-test.js index b781b5dd..46614465 100644 --- a/react-packager/src/Server/__tests__/Server-test.js +++ b/react-packager/src/Server/__tests__/Server-test.js @@ -87,6 +87,7 @@ describe('processRequest', () => { jest.fn().mockReturnValue({ getDependecyGraph: jest.fn().mockReturnValue({ getHasteMap: jest.fn().mockReturnValue({on: jest.fn()}), + load: jest.fn(() => Promise.resolve()), }), }); diff --git a/react-packager/src/Server/index.js b/react-packager/src/Server/index.js index d82fa80d..23f69495 100644 --- a/react-packager/src/Server/index.js +++ b/react-packager/src/Server/index.js @@ -241,14 +241,14 @@ class Server { this._fileWatcher.on('all', this._onFileChange.bind(this)); // changes to the haste map can affect resolution of files in the bundle - this._bundler - .getResolver() - .getDependecyGraph() - .getHasteMap() - .on('change', () => { + const dependencyGraph = this._bundler.getResolver().getDependecyGraph(); + + dependencyGraph.load().then(() => { + dependencyGraph.getHasteMap().on('change', () => { debug('Clearing bundle cache due to haste map change'); this._clearBundles(); }); + }); this._debouncedFileChangeHandler = debounceAndBatch(filePaths => { // only clear bundles for non-JS changes @@ -292,7 +292,7 @@ class Server { } buildBundle(options) { - return Promise.resolve().then(() => { + return this._bundler.getResolver().getDependecyGraph().load().then(() => { if (!options.platform) { options.platform = getPlatformExtension(options.entryFile); } @@ -692,7 +692,11 @@ class Server { } }, error => this._handleError(res, JSON.stringify(options), error) - ).done(); + ).catch(error => { + process.nextTick(() => { + throw error; + }); + }); } _symbolicate(req, res) { diff --git a/react-packager/src/node-haste/DependencyGraph/DeprecatedAssetMap.js b/react-packager/src/node-haste/DependencyGraph/DeprecatedAssetMap.js index 3cbe6419..dffccd01 100644 --- a/react-packager/src/node-haste/DependencyGraph/DeprecatedAssetMap.js +++ b/react-packager/src/node-haste/DependencyGraph/DeprecatedAssetMap.js @@ -9,73 +9,22 @@ 'use strict'; const AssetModule_DEPRECATED = require('../AssetModule_DEPRECATED'); -const Fastfs = require('../fastfs'); const debug = require('debug')('ReactNativePackager:DependencyGraph'); const path = require('../fastpath'); class DeprecatedAssetMap { constructor({ - fsCrawl, - roots, assetExts, - fileWatcher, - ignoreFilePath, helpers, - activity, - enabled, platforms, + files, }) { - if (roots == null || roots.length === 0 || !enabled) { - this._disabled = true; - return; - } - this._helpers = helpers; this._map = Object.create(null); this._assetExts = assetExts; - this._activity = activity; this._platforms = platforms; - if (!this._disabled) { - this._fastfs = new Fastfs( - 'Assets', - roots, - fileWatcher, - { ignore: ignoreFilePath, crawling: fsCrawl, activity } - ); - - this._fastfs.on('change', this._processFileChange.bind(this)); - } - } - - build() { - if (this._disabled) { - return Promise.resolve(); - } - - return this._fastfs.build().then( - () => { - const activity = this._activity; - let processAsset_DEPRECATEDActivity; - if (activity) { - processAsset_DEPRECATEDActivity = activity.startEvent( - 'Building (deprecated) Asset Map', - null, - { - telemetric: true, - }, - ); - } - - this._fastfs.findFilesByExts(this._assetExts).forEach( - file => this._processAsset(file) - ); - - if (activity) { - activity.endEvent(processAsset_DEPRECATEDActivity); - } - } - ); + files.forEach(file => this._processAsset(file)); } resolve(fromModule, toModuleName) { @@ -105,7 +54,7 @@ class DeprecatedAssetMap { } } - _processFileChange(type, filePath, root, fstat) { + processFileChange(type, filePath, root, fstat) { const name = assetName(filePath); if (type === 'change' || type === 'delete') { delete this._map[name]; diff --git a/react-packager/src/node-haste/__mocks__/graceful-fs.js b/react-packager/src/node-haste/__mocks__/graceful-fs.js index 1cfcb5cf..e72e4b0a 100644 --- a/react-packager/src/node-haste/__mocks__/graceful-fs.js +++ b/react-packager/src/node-haste/__mocks__/graceful-fs.js @@ -77,6 +77,15 @@ fs.readFile.mockImpl(function(filepath, encoding, callback) { } }); +fs.readFileSync.mockImpl(function(filepath, encoding) { + const node = getToNode(filepath); + // dir check + if (node && typeof node === 'object' && node.SYMLINK == null) { + throw new Error('Error readFileSync a dir: ' + filepath); + } + return node; +}); + fs.stat.mockImpl((filepath, callback) => { callback = asyncCallback(callback); let node; @@ -121,6 +130,31 @@ fs.statSync.mockImpl((filepath) => { }; }); +fs.lstat.mockImpl((filepath, callback) => { + callback = asyncCallback(callback); + let node; + try { + node = getToNode(filepath); + } catch (e) { + callback(e); + return; + } + + if (node && typeof node === 'object') { + callback(null, { + isDirectory: () => true, + isSymbolicLink: () => false, + mtime, + }); + } else { + callback(null, { + isDirectory: () => false, + isSymbolicLink: () => false, + mtime, + }); + } +}); + fs.lstatSync.mockImpl((filepath) => { const node = getToNode(filepath); diff --git a/react-packager/src/node-haste/__tests__/DependencyGraph-test.js b/react-packager/src/node-haste/__tests__/DependencyGraph-test.js index eac88cdd..02bb06f2 100644 --- a/react-packager/src/node-haste/__tests__/DependencyGraph-test.js +++ b/react-packager/src/node-haste/__tests__/DependencyGraph-test.js @@ -12,6 +12,99 @@ jest.autoMockOff(); jest.useRealTimers(); jest.mock('fs'); +// This is an ugly hack: +// * jest-haste-map uses `find` for fast file system crawling which won't work +// when we mock the file system in node. This mock copies the node crawler's +// implementation and always falls back to the node crawling mechanism. +// Ideally we'll make this an option in jest-haste-map to force it to use +// the node crawler. +jest.mock('jest-haste-map/build/crawlers/node', () => { + const H = require('jest-haste-map/build/constants'); + + const fs = require('fs'); + const path = require('path'); + + function find( + roots, + extensions, + ignore, + callback) + { + const result = []; + let activeCalls = 0; + + function search(directory) { + activeCalls++; + fs.readdir(directory, (err, names) => { + activeCalls--; + + names.forEach(file => { + file = process.platform === 'win32' ? + path.win32.join(directory, file) : + path.join(directory, file); + if (ignore(file)) { + return; + } + activeCalls++; + + fs.lstat(file, (err, stat) => { + activeCalls--; + + if (!err && stat && !stat.isSymbolicLink()) { + if (stat.isDirectory()) { + search(file); + } else { + const ext = path.extname(file).substr(1); + if (extensions.indexOf(ext) !== -1) { + result.push([file, stat.mtime.getTime()]); + } + } + } + if (activeCalls === 0) { + callback(result); + } + }); + }); + + if (activeCalls === 0) { + callback(result); + } + }); + } + + roots.forEach(search); + } + + return function nodeCrawl( + roots, + extensions, + ignore, + data) + { + return new Promise(resolve => { + const callback = list => { + const files = Object.create(null); + list.forEach(fileData => { + const name = fileData[0]; + const mtime = fileData[1]; + const existingFile = data.files[name]; + if (existingFile && existingFile[H.MTIME] === mtime) { + files[name] = existingFile; + } else { + // See ../constants.js + files[name] = ['', mtime, 0, []]; + } + }); + data.files = files; + resolve(data); + }; + + find(roots, extensions, ignore, callback); + }); + }; + +}); + const mocksPattern = /(?:[\\/]|^)__mocks__[\\/]([^\/]+)\.js$/; jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000; @@ -103,11 +196,15 @@ describe('DependencyGraph', function() { ], platforms: ['ios', 'android'], shouldThrowOnUnresolvedErrors: () => false, + useWatchman: false, + maxWorkers: 1, + resetCache: true, }; }); describe('get sync dependencies (posix)', function() { let DependencyGraph; + const consoleWarn = console.warn; const realPlatform = process.platform; beforeEach(function() { process.platform = 'linux'; @@ -115,10 +212,11 @@ describe('DependencyGraph', function() { }); afterEach(function() { + console.warn = consoleWarn; process.platform = realPlatform; }); - pit('should get dependencies', function() { + it('should get dependencies', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -186,7 +284,7 @@ describe('DependencyGraph', function() { }); }); - pit('should resolve relative entry path', function() { + it('should resolve relative entry path', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -219,7 +317,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get shallow dependencies', function() { + it('should get shallow dependencies', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -274,7 +372,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get dependencies with the correct extensions', function() { + it('should get dependencies with the correct extensions', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -328,7 +426,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get json dependencies', function() { + it('should get json dependencies', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -388,7 +486,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get package json as a dep', () => { + it('should get package json as a dep', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -435,7 +533,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get dependencies with deprecated assets', function() { + it('should get dependencies with deprecated assets', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -483,7 +581,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get dependencies with relative assets', function() { + it('should get dependencies with relative assets', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -533,7 +631,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get dependencies with assets and resolution', function() { + it('should get dependencies with assets and resolution', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -612,7 +710,7 @@ describe('DependencyGraph', function() { }); }); - pit('should respect platform extension in assets', function() { + it('should respect platform extension in assets', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -692,7 +790,7 @@ describe('DependencyGraph', function() { }); }); - pit('Deprecated and relative assets can live together', function() { + it('Deprecated and relative assets can live together', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -754,7 +852,7 @@ describe('DependencyGraph', function() { }); }); - pit('should get recursive dependencies', function() { + it('should get recursive dependencies', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -804,7 +902,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with packages', function() { + it('should work with packages', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -855,7 +953,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with packages', function() { + it('should work with packages', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -906,7 +1004,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with packages with a dot in the name', function() { + it('should work with packages with a dot in the name', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -975,7 +1073,7 @@ describe('DependencyGraph', function() { }); }); - pit('should default main package to index.js', function() { + it('should default main package to index.js', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1020,7 +1118,7 @@ describe('DependencyGraph', function() { }); }); - pit('should resolve using alternative ids', () => { + it('should resolve using alternative ids', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -1069,7 +1167,7 @@ describe('DependencyGraph', function() { }); }); - pit('should default use index.js if main is a dir', function() { + it('should default use index.js if main is a dir', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1117,7 +1215,7 @@ describe('DependencyGraph', function() { }); }); - pit('should resolve require to index if it is a dir', function() { + it('should resolve require to index if it is a dir', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1162,7 +1260,7 @@ describe('DependencyGraph', function() { }); }); - pit('should resolve require to main if it is a dir w/ a package.json', function() { + it('should resolve require to main if it is a dir w/ a package.json', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1211,7 +1309,7 @@ describe('DependencyGraph', function() { }); }); - pit('should ignore malformed packages', function() { + it('should ignore malformed packages', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1248,8 +1346,9 @@ describe('DependencyGraph', function() { }); }); - pit('should fatal on multiple modules with the same name', function() { - var root = '/root'; + it('should fatal on multiple modules with the same name', function() { + const root = '/root'; + console.warn = jest.fn(); setMockFileSystem({ 'root': { 'index.js': [ @@ -1279,10 +1378,11 @@ describe('DependencyGraph', function() { 'with the same name across two different files.' ); expect(err.type).toEqual('DependencyGraphError'); + expect(console.warn).toBeCalled(); }); }); - pit('should be forgiving with missing requires', function() { + it('should be forgiving with missing requires', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1317,7 +1417,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with packages with subdirs', function() { + it('should work with packages with subdirs', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1373,7 +1473,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with packages with symlinked subdirs', function() { + it('should work with packages with symlinked subdirs', function() { var root = '/root'; setMockFileSystem({ 'symlinkedPackage': { @@ -1430,7 +1530,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with relative modules in packages', function() { + it('should work with relative modules in packages', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1522,7 +1622,7 @@ describe('DependencyGraph', function() { } function testBrowserField(fieldName) { - pit('should support simple browser field in packages ("' + fieldName + '")', function() { + it('should support simple browser field in packages ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1577,7 +1677,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() { + it('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1630,7 +1730,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support mapping main in browser field json ("' + fieldName + '")', function() { + it('should support mapping main in browser field json ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1686,7 +1786,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() { + it('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1744,7 +1844,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser mapping of files ("' + fieldName + '")', function() { + it('should support browser mapping of files ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1848,7 +1948,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser mapping for packages ("' + fieldName + '")', function() { + it('should support browser mapping for packages ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -1920,7 +2020,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser mapping of a package to a file ("' + fieldName + '")', () => { + it('should support browser mapping of a package to a file ("' + fieldName + '")', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -1999,7 +2099,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser mapping for packages ("' + fieldName + '")', function() { + it('should support browser mapping for packages ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2071,7 +2171,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser exclude of a package ("' + fieldName + '")', function() { + it('should support browser exclude of a package ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2128,7 +2228,7 @@ describe('DependencyGraph', function() { }); }); - pit('should support browser exclude of a file ("' + fieldName + '")', function() { + it('should support browser exclude of a file ("' + fieldName + '")', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2181,7 +2281,7 @@ describe('DependencyGraph', function() { }); } - pit('should fall back to browser mapping from react-native mapping', function() { + it('should fall back to browser mapping from react-native mapping', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2273,7 +2373,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with absolute paths', () => { + it('should work with absolute paths', () => { const root = '/root'; setMockFileSystem({ [root.slice(1)]: { @@ -2313,7 +2413,7 @@ describe('DependencyGraph', function() { }); }); - pit('should merge browser mapping with react-native mapping', function() { + it('should merge browser mapping with react-native mapping', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2451,7 +2551,7 @@ describe('DependencyGraph', function() { }); }); - pit('should fall back to `extraNodeModules`', () => { + it('should fall back to `extraNodeModules`', () => { const root = '/root'; setMockFileSystem({ [root.slice(1)]: { @@ -2513,7 +2613,7 @@ describe('DependencyGraph', function() { }); }); - pit( + it( 'should only use `extraNodeModules` after checking all possible filesystem locations', () => { const root = '/root'; @@ -2561,7 +2661,7 @@ describe('DependencyGraph', function() { } ); - pit('should be able to resolve paths within `extraNodeModules`', () => { + it('should be able to resolve paths within `extraNodeModules`', () => { const root = '/root'; setMockFileSystem({ [root.slice(1)]: { @@ -2692,7 +2792,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with absolute paths', () => { + it('should work with absolute paths', () => { const root = 'C:\\root'; setMockFileSystem({ 'root': { @@ -2745,7 +2845,7 @@ describe('DependencyGraph', function() { process.platform = realPlatform; }); - pit('should work with nested node_modules', function() { + it('should work with nested node_modules', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2835,7 +2935,7 @@ describe('DependencyGraph', function() { }); }); - pit('platform should work with node_modules', function() { + it('platform should work with node_modules', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2905,7 +3005,7 @@ describe('DependencyGraph', function() { }); }); - pit('nested node_modules with specific paths', function() { + it('nested node_modules with specific paths', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -2996,7 +3096,7 @@ describe('DependencyGraph', function() { }); }); - pit('nested node_modules with browser field', function() { + it('nested node_modules with browser field', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3091,7 +3191,7 @@ describe('DependencyGraph', function() { }); }); - pit('node_modules should support multi level', function() { + it('node_modules should support multi level', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3165,7 +3265,7 @@ describe('DependencyGraph', function() { }); }); - pit('should selectively ignore providesModule in node_modules', function() { + it('should selectively ignore providesModule in node_modules', function() { var root = '/root'; var otherRoot = '/anotherRoot'; setMockFileSystem({ @@ -3339,7 +3439,7 @@ describe('DependencyGraph', function() { }); }); - pit('should not be confused by prev occuring whitelisted names', function() { + it('should not be confused by prev occuring whitelisted names', function() { var root = '/react-haste'; setMockFileSystem({ 'react-haste': { @@ -3397,7 +3497,7 @@ describe('DependencyGraph', function() { }); - pit('should ignore modules it cant find (assumes own require system)', function() { + it('should ignore modules it cant find (assumes own require system)', function() { // For example SourceMap.js implements it's own require system. var root = '/root'; setMockFileSystem({ @@ -3441,7 +3541,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with node packages with a .js in the name', function() { + it('should work with node packages with a .js in the name', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3494,7 +3594,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with multiple platforms (haste)', function() { + it('should work with multiple platforms (haste)', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3553,7 +3653,7 @@ describe('DependencyGraph', function() { }); }); - pit('should pick the generic file', function() { + it('should pick the generic file', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3613,7 +3713,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with multiple platforms (node)', function() { + it('should work with multiple platforms (node)', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3660,7 +3760,7 @@ describe('DependencyGraph', function() { }); }); - pit('should require package.json', () => { + it('should require package.json', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -3741,7 +3841,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with one-character node_modules', () => { + it('should work with one-character node_modules', () => { const root = '/root'; setMockFileSystem({ [root.slice(1)]: { @@ -3796,7 +3896,7 @@ describe('DependencyGraph', function() { const DependencyGraph = require('../index'); - pit('should work with nested node_modules', function() { + it('should work with nested node_modules', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3886,7 +3986,7 @@ describe('DependencyGraph', function() { }); }); - pit('platform should work with node_modules', function() { + it('platform should work with node_modules', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -3956,7 +4056,7 @@ describe('DependencyGraph', function() { }); }); - pit('nested node_modules with specific paths', function() { + it('nested node_modules with specific paths', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4047,7 +4147,7 @@ describe('DependencyGraph', function() { }); }); - pit('nested node_modules with browser field', function() { + it('nested node_modules with browser field', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4142,7 +4242,7 @@ describe('DependencyGraph', function() { }); }); - pit('node_modules should support multi level', function() { + it('node_modules should support multi level', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4216,7 +4316,7 @@ describe('DependencyGraph', function() { }); }); - pit('should selectively ignore providesModule in node_modules', function() { + it('should selectively ignore providesModule in node_modules', function() { var root = '/root'; var otherRoot = '/anotherRoot'; setMockFileSystem({ @@ -4390,7 +4490,7 @@ describe('DependencyGraph', function() { }); }); - pit('should not be confused by prev occuring whitelisted names', function() { + it('should not be confused by prev occuring whitelisted names', function() { var root = '/react-haste'; setMockFileSystem({ 'react-haste': { @@ -4447,7 +4547,7 @@ describe('DependencyGraph', function() { }); }); - pit('should ignore modules it cant find (assumes own require system)', function() { + it('should ignore modules it cant find (assumes own require system)', function() { // For example SourceMap.js implements it's own require system. var root = '/root'; setMockFileSystem({ @@ -4491,7 +4591,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with node packages with a .js in the name', function() { + it('should work with node packages with a .js in the name', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4544,7 +4644,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with multiple platforms (haste)', function() { + it('should work with multiple platforms (haste)', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4603,7 +4703,7 @@ describe('DependencyGraph', function() { }); }); - pit('should pick the generic file', function() { + it('should pick the generic file', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4662,7 +4762,7 @@ describe('DependencyGraph', function() { }); }); - pit('should work with multiple platforms (node)', function() { + it('should work with multiple platforms (node)', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -4709,7 +4809,7 @@ describe('DependencyGraph', function() { }); }); - pit('should require package.json', () => { + it('should require package.json', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -4824,7 +4924,7 @@ describe('DependencyGraph', function() { process.platform = realPlatform; }); - pit('updates module dependencies', function() { + it('updates module dependencies', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -4887,7 +4987,7 @@ describe('DependencyGraph', function() { }); }); - pit('updates module dependencies on file change', function() { + it('updates module dependencies on file change', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -4950,7 +5050,7 @@ describe('DependencyGraph', function() { }); }); - pit('updates module dependencies on file delete', function() { + it('updates module dependencies on file delete', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5012,7 +5112,7 @@ describe('DependencyGraph', function() { }); }); - pit('updates module dependencies on file add', function() { + it('updates module dependencies on file add', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5105,7 +5205,7 @@ describe('DependencyGraph', function() { }); }); - pit('updates module dependencies on deprecated asset add', function() { + it('updates module dependencies on deprecated asset add', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5174,7 +5274,7 @@ describe('DependencyGraph', function() { }); }); - pit('updates module dependencies on relative asset add', function() { + it('updates module dependencies on relative asset add', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5244,7 +5344,7 @@ describe('DependencyGraph', function() { }); }); - pit('runs changes through ignore filter', function() { + it('runs changes through ignore filter', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5334,7 +5434,7 @@ describe('DependencyGraph', function() { }); }); - pit('should ignore directory updates', function() { + it('should ignore directory updates', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -5409,7 +5509,7 @@ describe('DependencyGraph', function() { }); }); - pit('changes to browser field', function() { + it('changes to browser field', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5472,7 +5572,7 @@ describe('DependencyGraph', function() { }); }); - pit('removes old package from cache', function() { + it('removes old package from cache', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5523,7 +5623,7 @@ describe('DependencyGraph', function() { }); }); - pit('should update node package changes', function() { + it('should update node package changes', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5629,7 +5729,7 @@ describe('DependencyGraph', function() { }); }); - pit('should update node package main changes', function() { + it('should update node package main changes', function() { var root = '/root'; var filesystem = setMockFileSystem({ 'root': { @@ -5694,7 +5794,7 @@ describe('DependencyGraph', function() { }); }); - pit('should not error when the watcher reports a known file as added', function() { + it('should not error when the watcher reports a known file as added', function() { var root = '/root'; setMockFileSystem({ 'root': { @@ -5737,7 +5837,7 @@ describe('DependencyGraph', function() { process.platform = realPlatform; }); - pit('supports custom file extensions', () => { + it('supports custom file extensions', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -5808,7 +5908,7 @@ describe('DependencyGraph', function() { process.platform = realPlatform; }); - pit('resolves to null if mocksPattern is not specified', () => { + it('resolves to null if mocksPattern is not specified', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -5830,7 +5930,7 @@ describe('DependencyGraph', function() { }); }); - pit('retrieves a list of all required mocks', () => { + it('retrieves a list of all required mocks', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -5863,7 +5963,7 @@ describe('DependencyGraph', function() { }); }); - pit('adds mocks as a dependency of their actual module', () => { + it('adds mocks as a dependency of their actual module', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -5936,7 +6036,7 @@ describe('DependencyGraph', function() { }); }); - pit('resolves mocks that do not have a real module associated with them', () => { + it('resolves mocks that do not have a real module associated with them', () => { var root = '/root'; setMockFileSystem({ 'root': { @@ -6035,20 +6135,20 @@ describe('DependencyGraph', function() { }); }); - pit('calls back for each finished module', () => { + it('calls back for each finished module', () => { return getDependencies().then(() => expect(onProgress.mock.calls.length).toBe(8) ); }); - pit('increases the number of finished modules in steps of one', () => { + it('increases the number of finished modules in steps of one', () => { return getDependencies().then(() => { const increments = onProgress.mock.calls.map(([finished]) => finished); expect(increments).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); }); }); - pit('adds the number of discovered modules to the number of total modules', () => { + it('adds the number of discovered modules to the number of total modules', () => { return getDependencies().then(() => { const increments = onProgress.mock.calls.map(([, total]) => total); expect(increments).toEqual([3, 5, 6, 6, 7, 7, 8, 8]); @@ -6062,7 +6162,7 @@ describe('DependencyGraph', function() { DependencyGraph = require('../index'); }); - pit('allows setting dependencies for asset modules', () => { + it('allows setting dependencies for asset modules', () => { const assetDependencies = ['arbitrary', 'dependencies']; setMockFileSystem({ @@ -6121,7 +6221,7 @@ describe('DependencyGraph', function() { roots: ['/root'], }); moduleReadDeferreds = {}; - callDeferreds = [defer()/* a.js */, defer()/* b.js */]; + callDeferreds = [defer(), defer()]; // [a.js, b.js] Module.prototype.read = jest.genMockFn().mockImplementation(function() { const returnValue = moduleRead.apply(this, arguments); @@ -6143,7 +6243,7 @@ describe('DependencyGraph', function() { Module.prototype.read = moduleRead; }); - pit('produces a deterministic tree if the "a" module resolves first', () => { + it('produces a deterministic tree if the "a" module resolves first', () => { const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js'); return Promise.all(callDeferreds.map(deferred => deferred.promise)) @@ -6168,7 +6268,7 @@ describe('DependencyGraph', function() { }); }); - pit('produces a deterministic tree if the "b" module resolves first', () => { + it('produces a deterministic tree if the "b" module resolves first', () => { const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js'); return Promise.all(callDeferreds.map(deferred => deferred.promise)) diff --git a/react-packager/src/node-haste/__tests__/Module-test.js b/react-packager/src/node-haste/__tests__/Module-test.js index 0e9c75c4..1d5399ee 100644 --- a/react-packager/src/node-haste/__tests__/Module-test.js +++ b/react-packager/src/node-haste/__tests__/Module-test.js @@ -78,17 +78,16 @@ describe('Module', () => { const createJSONModule = (options) => createModule({...options, file: '/root/package.json'}); - beforeEach(function(done) { + beforeEach(function() { process.platform = 'linux'; cache = createCache(); fastfs = new Fastfs( 'test', ['/root'], fileWatcher, - {crawling: Promise.resolve([fileName, '/root/package.json']), ignore: []}, + ['/root/index.js', '/root/package.json'], + {ignore: []}, ); - - fastfs.build().then(done); }); describe('Module ID', () => { diff --git a/react-packager/src/node-haste/__tests__/fastfs-integrated-test.js b/react-packager/src/node-haste/__tests__/fastfs-integrated-test.js index cbff314a..e99be2e4 100644 --- a/react-packager/src/node-haste/__tests__/fastfs-integrated-test.js +++ b/react-packager/src/node-haste/__tests__/fastfs-integrated-test.js @@ -22,13 +22,17 @@ const contents = fs.readFileSync(fileName, 'utf-8'); describe('fastfs:', function() { let fastfs; - const crawling = Promise.resolve([fileName]); const roots = [__dirname]; const watcher = new EventEmitter(); - beforeEach(function(done) { - fastfs = new Fastfs('arbitrary', roots, watcher, {crawling}); - fastfs.build().then(done); + beforeEach(function() { + fastfs = new Fastfs( + 'arbitrary', + roots, + watcher, + [`${__dirname}/fastfs-data`], + {} + ); }); describe('partial reading', () => { diff --git a/react-packager/src/node-haste/crawlers/index.js b/react-packager/src/node-haste/crawlers/index.js deleted file mode 100644 index 16b73f27..00000000 --- a/react-packager/src/node-haste/crawlers/index.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict'; - -const nodeCrawl = require('./node'); -const watchmanCrawl = require('./watchman'); - -function crawl(roots, options) { - const {fileWatcher} = options; - return (fileWatcher ? fileWatcher.isWatchman() : Promise.resolve(false)).then( - isWatchman => isWatchman ? watchmanCrawl(roots, options) : nodeCrawl(roots, options) - ); -} - -module.exports = crawl; diff --git a/react-packager/src/node-haste/crawlers/node.js b/react-packager/src/node-haste/crawlers/node.js deleted file mode 100644 index cb018ded..00000000 --- a/react-packager/src/node-haste/crawlers/node.js +++ /dev/null @@ -1,61 +0,0 @@ -'use strict'; - -const denodeify = require('denodeify'); -const debug = require('debug')('ReactNativePackager:DependencyGraph'); -const fs = require('graceful-fs'); -const path = require('../fastpath'); - -const readDir = denodeify(fs.readdir); -const stat = denodeify(fs.stat); - -function nodeRecReadDir(roots, {ignore, exts}) { - const queue = roots.slice(); - const retFiles = []; - const extPattern = new RegExp( - '\.(' + exts.join('|') + ')$' - ); - - function search() { - const currDir = queue.shift(); - if (!currDir) { - return Promise.resolve(); - } - - return readDir(currDir) - .then(files => files.map(f => path.join(currDir, f))) - .then(files => Promise.all( - files.map(f => stat(f).catch(handleBrokenLink)) - ).then(stats => [ - // Remove broken links. - files.filter((file, i) => !!stats[i]), - stats.filter(Boolean), - ])) - .then(([files, stats]) => { - files.forEach((filePath, i) => { - if (ignore(filePath)) { - return; - } - - if (stats[i].isDirectory()) { - queue.push(filePath); - return; - } - - if (filePath.match(extPattern)) { - retFiles.push(path.resolve(filePath)); - } - }); - - return search(); - }); - } - - return search().then(() => retFiles); -} - -function handleBrokenLink(e) { - debug('WARNING: error stating, possibly broken symlink', e.message); - return Promise.resolve(); -} - -module.exports = nodeRecReadDir; diff --git a/react-packager/src/node-haste/crawlers/watchman.js b/react-packager/src/node-haste/crawlers/watchman.js deleted file mode 100644 index 284fe673..00000000 --- a/react-packager/src/node-haste/crawlers/watchman.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict'; - -const denodeify = require('denodeify'); -const path = require('../fastpath'); - -const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html'; - -function watchmanRecReadDir(roots, {ignore, fileWatcher, exts}) { - const files = []; - return Promise.all( - roots.map( - root => fileWatcher.getWatcherForRoot(root) - ) - ).then( - watchers => { - // All watchman roots for all watches we have. - const watchmanRoots = watchers.map( - watcher => watcher.watchProjectInfo.root - ); - - // Actual unique watchers (because we use watch-project we may end up with - // duplicate "real" watches, and that's by design). - // TODO(amasad): push this functionality into the `FileWatcher`. - const uniqueWatchers = watchers.filter( - (watcher, i) => watchmanRoots.indexOf(watcher.watchProjectInfo.root) === i - ); - - return Promise.all( - uniqueWatchers.map(watcher => { - const watchedRoot = watcher.watchProjectInfo.root; - - // Build up an expression to filter the output by the relevant roots. - const dirExpr = ['anyof']; - for (let i = 0; i < roots.length; i++) { - const root = roots[i]; - if (isDescendant(watchedRoot, root)) { - dirExpr.push(['dirname', path.relative(watchedRoot, root)]); - } - } - - const cmd = denodeify(watcher.client.command.bind(watcher.client)); - return cmd(['query', watchedRoot, { - suffix: exts, - expression: ['allof', ['type', 'f'], 'exists', dirExpr], - fields: ['name'], - }]).then(resp => { - if ('warning' in resp) { - console.warn('watchman warning: ', resp.warning); - } - - resp.files.forEach(filePath => { - filePath = watchedRoot + path.sep + filePath; - if (!ignore(filePath)) { - files.push(filePath); - } - return false; - }); - }); - }) - ); - }).then( - () => files, - error => { - throw new Error( - `Watchman error: ${error.message.trim()}. Make sure watchman ` + - `is running for this project. See ${watchmanURL}.` - ); - } - ); -} - -function isDescendant(root, child) { - return root === child || child.startsWith(root + path.sep); -} - -module.exports = watchmanRecReadDir; diff --git a/react-packager/src/node-haste/fastfs.js b/react-packager/src/node-haste/fastfs.js index ad25c1d5..492b6fbd 100644 --- a/react-packager/src/node-haste/fastfs.js +++ b/react-packager/src/node-haste/fastfs.js @@ -20,7 +20,7 @@ const stat = denodeify(fs.stat); const NOT_FOUND_IN_ROOTS = 'NotFoundInRootsError'; class Fastfs extends EventEmitter { - constructor(name, roots, fileWatcher, {ignore, crawling, activity}) { + constructor(name, roots, fileWatcher, files, {ignore, activity}) { super(); this._name = name; this._fileWatcher = fileWatcher; @@ -37,45 +37,39 @@ class Fastfs extends EventEmitter { return new File(root, true); }); this._fastPaths = Object.create(null); - this._crawling = crawling; this._activity = activity; - } - build() { - return this._crawling.then(files => { - let fastfsActivity; - const activity = this._activity; - if (activity) { - fastfsActivity = activity.startEvent( - 'Building in-memory fs for ' + this._name, - null, - { - telemetric: true, - }, - ); - } - files.forEach(filePath => { - const root = this._getRoot(filePath); - if (root) { - const newFile = new File(filePath, false); - const dirname = filePath.substr(0, filePath.lastIndexOf(path.sep)); - const parent = this._fastPaths[dirname]; - this._fastPaths[filePath] = newFile; - if (parent) { - parent.addChild(newFile, this._fastPaths); - } else { - root.addChild(newFile, this._fastPaths); - } + let fastfsActivity; + if (activity) { + fastfsActivity = activity.startEvent( + 'Building in-memory fs for ' + this._name, + null, + { + telemetric: true, + }, + ); + } + files.forEach(filePath => { + const root = this._getRoot(filePath); + if (root) { + const newFile = new File(filePath, false); + const dirname = filePath.substr(0, filePath.lastIndexOf(path.sep)); + const parent = this._fastPaths[dirname]; + this._fastPaths[filePath] = newFile; + if (parent) { + parent.addChild(newFile, this._fastPaths); + } else { + root.addChild(newFile, this._fastPaths); } - }); - if (activity) { - activity.endEvent(fastfsActivity); - } - - if (this._fileWatcher) { - this._fileWatcher.on('all', this._processFileChange.bind(this)); } }); + if (activity) { + activity.endEvent(fastfsActivity); + } + + if (this._fileWatcher) { + this._fileWatcher.on('all', this._processFileChange.bind(this)); + } } stat(filePath) { diff --git a/react-packager/src/node-haste/index.js b/react-packager/src/node-haste/index.js index 84f5290c..73a41df7 100644 --- a/react-packager/src/node-haste/index.js +++ b/react-packager/src/node-haste/index.js @@ -11,10 +11,10 @@ const Cache = require('./Cache'); const Fastfs = require('./fastfs'); const FileWatcher = require('./FileWatcher'); +const JestHasteMap = require('jest-haste-map'); const Module = require('./Module'); const ModuleCache = require('./ModuleCache'); const Polyfill = require('./Polyfill'); -const crawl = require('./crawlers'); const extractRequires = require('./lib/extractRequires'); const getAssetDataFromName = require('./lib/getAssetDataFromName'); const getInverseDependencies = require('./lib/getInverseDependencies'); @@ -23,6 +23,7 @@ const isAbsolutePath = require('absolute-path'); const replacePatterns = require('./lib/replacePatterns'); const path = require('./fastpath'); const util = require('util'); +const os = require('os'); const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers'); const ResolutionRequest = require('./DependencyGraph/ResolutionRequest'); const ResolutionResponse = require('./DependencyGraph/ResolutionResponse'); @@ -57,6 +58,10 @@ class DependencyGraph { assetDependencies, moduleOptions, extraNodeModules, + // additional arguments for jest-haste-map + useWatchman, + maxWorkers, + resetCache, }) { this._opts = { activity: activity || defaultActivity, @@ -78,6 +83,10 @@ class DependencyGraph { cacheTransformResults: true, }, extraNodeModules, + // additional arguments for jest-haste-map & defaults + useWatchman: useWatchman !== false, + maxWorkers, + resetCache, }; this._cache = cache; this._assetDependencies = assetDependencies; @@ -90,103 +99,110 @@ class DependencyGraph { return this._loading; } - const {activity} = this._opts; - const depGraphActivity = activity.startEvent( - 'Initializing Packager', - null, - { - telemetric: true, - }, - ); - const crawlActivity = activity.startEvent( - 'Crawling File System', - null, - { - telemetric: true, - }, - ); - const allRoots = this._opts.roots.concat(this._opts.assetRoots_DEPRECATED); - this._crawling = crawl(allRoots, { - ignore: this._opts.ignoreFilePath, - exts: this._opts.extensions.concat(this._opts.assetExts), - fileWatcher: this._opts.fileWatcher, - }); - this._crawling.then((files) => activity.endEvent(crawlActivity)); - - this._fastfs = new Fastfs( - 'JavaScript', - this._opts.roots, - this._opts.fileWatcher, - { - ignore: this._opts.ignoreFilePath, - crawling: this._crawling, - activity: activity, - } - ); - - this._fastfs.on('change', this._processFileChange.bind(this)); - - this._moduleCache = new ModuleCache({ - fastfs: this._fastfs, - cache: this._cache, - extractRequires: this._opts.extractRequires, - transformCode: this._opts.transformCode, - depGraphHelpers: this._helpers, - assetDependencies: this._assetDependencies, - moduleOptions: this._opts.moduleOptions, - }, this._opts.platforms); - - this._hasteMap = new HasteMap({ - fastfs: this._fastfs, - extensions: this._opts.extensions, - moduleCache: this._moduleCache, - preferNativePlatform: this._opts.preferNativePlatform, - helpers: this._helpers, - platforms: this._opts.platforms, + const mw = this._opts.maxWorkers; + const haste = new JestHasteMap({ + extensions: this._opts.extensions.concat(this._opts.assetExts), + ignorePattern: {test: this._opts.ignoreFilePath}, + maxWorkers: typeof mw === 'number' && mw >= 1 ? mw : getMaxWorkers(), + mocksPattern: '', + name: 'react-native-packager', + platforms: Array.from(this._opts.platforms), + providesModuleNodeModules: this._opts.providesModuleNodeModules, + resetCache: this._opts.resetCache, + retainAllFiles: true, + roots: this._opts.roots.concat(this._opts.assetRoots_DEPRECATED), + useWatchman: this._opts.useWatchman, }); - this._deprecatedAssetMap = new DeprecatedAssetMap({ - fsCrawl: this._crawling, - roots: this._opts.assetRoots_DEPRECATED, - helpers: this._helpers, - fileWatcher: this._opts.fileWatcher, - ignoreFilePath: this._opts.ignoreFilePath, - assetExts: this._opts.assetExts, - activity: this._opts.activity, - enabled: this._opts.enableAssetMap, - platforms: this._opts.platforms, - }); + this._loading = haste.build().then(hasteMap => { + const {activity} = this._opts; + const depGraphActivity = activity.startEvent( + 'Initializing Packager', + null, + { + telemetric: true, + }, + ); - this._loading = Promise.all([ - this._fastfs.build() - .then(() => { - const hasteActivity = activity.startEvent( - 'Building Haste Map', - null, - { - telemetric: true, - }, + const hasteFSFiles = hasteMap.hasteFS.getAllFiles(); + + this._fastfs = new Fastfs( + 'JavaScript', + this._opts.roots, + this._opts.fileWatcher, + hasteFSFiles, + { + ignore: this._opts.ignoreFilePath, + activity: activity, + } + ); + + this._fastfs.on('change', this._processFileChange.bind(this)); + + this._moduleCache = new ModuleCache({ + fastfs: this._fastfs, + cache: this._cache, + extractRequires: this._opts.extractRequires, + transformCode: this._opts.transformCode, + depGraphHelpers: this._helpers, + assetDependencies: this._assetDependencies, + moduleOptions: this._opts.moduleOptions, + }, this._opts.platforms); + + this._hasteMap = new HasteMap({ + fastfs: this._fastfs, + extensions: this._opts.extensions, + moduleCache: this._moduleCache, + preferNativePlatform: this._opts.preferNativePlatform, + helpers: this._helpers, + platforms: this._opts.platforms, + }); + + const escapePath = (p: string) => { + return (path.sep === '\\') ? p.replace(/(\/|\\(?!\.))/g, '\\\\') : p; + }; + + const assetPattern = + new RegExp('^' + this._opts.assetRoots_DEPRECATED.map(escapePath).join('|')); + + const assetFiles = hasteMap.hasteFS.matchFiles(assetPattern); + + this._deprecatedAssetMap = new DeprecatedAssetMap({ + helpers: this._helpers, + assetExts: this._opts.assetExts, + platforms: this._opts.platforms, + files: assetFiles, + }); + + this._fastfs.on('change', (type, filePath, root, fstat) => { + if (assetPattern.test(path.join(root, filePath))) { + this._deprecatedAssetMap.processFileChange(type, filePath, root, fstat); + } + }); + + const hasteActivity = activity.startEvent( + 'Building Haste Map', + null, + { + telemetric: true, + }, + ); + return this._hasteMap.build().then( + map => { + activity.endEvent(hasteActivity); + activity.endEvent(depGraphActivity); + return map; + }, + err => { + const error = new Error( + `Failed to build DependencyGraph: ${err.message}` ); - return this._hasteMap.build().then(map => { - activity.endEvent(hasteActivity); - return map; - }); - }), - this._deprecatedAssetMap.build(), - ]).then( - response => { - activity.endEvent(depGraphActivity); - return response[0]; // Return the haste map - }, - err => { - const error = new Error( - `Failed to build DependencyGraph: ${err.message}` - ); - error.type = ERROR_BUILDING_DEP_GRAPH; - error.stack = err.stack; - throw error; - } - ); + error.type = ERROR_BUILDING_DEP_GRAPH; + error.stack = err.stack; + throw error; + } + ); + }); return this._loading; } @@ -312,7 +328,7 @@ class DependencyGraph { this._loading = this._hasteMap.build(); } else { this._loading = this._hasteMap.processFileChange(type, absPath); - this._loading.catch((e) => this._hasteMapError = e); + this._loading.catch((e) => {this._hasteMapError = e;}); } return this._loading; }; @@ -351,4 +367,28 @@ function NotFoundError() { } util.inherits(NotFoundError, Error); +function getMaxWorkers() { + const cores = os.cpus().length; + + if (cores <= 1) { + // oh well... + return 1; + } + if (cores <= 4) { + // don't starve the CPU while still reading reasonably rapidly + return cores - 1; + } + if (cores <= 8) { + // empirical testing showed massive diminishing returns when going over + // 4 or 5 workers on 8-core machines + return Math.floor(cores * 0.75) - 1; + } + + // pretty much guesswork + if (cores < 24) { + return Math.floor(3 / 8 * cores + 3); + } + return cores / 2; +} + module.exports = DependencyGraph;