Replacing node-haste with jest-haste-map

Summary: Modified `node-haste` implementation to use the much faster `jest-haste-map` under the hood. The underlying `fastfs` now gets passed the entire file list from the `jest-haste-map` rather than crawl the filesystem.

Reviewed By: cpojer

Differential Revision: D3724387

fbshipit-source-id: 447d58ea0edf283662ec23d1e2deee992cf8d240
This commit is contained in:
Ovidiu Viorel Iepure 2016-10-03 05:16:04 -07:00 committed by Facebook Github Bot
parent d7d89172c2
commit 6130650d93
15 changed files with 436 additions and 453 deletions

View File

@ -166,6 +166,7 @@
"image-size": "^0.3.5",
"immutable": "~3.7.6",
"inquirer": "^0.12.0",
"jest-haste-map": "15.0.1",
"joi": "^6.6.1",
"json-stable-stringify": "^1.0.1",
"json5": "^0.4.0",

View File

@ -143,19 +143,20 @@ class Bundler {
});
this._resolver = new Resolver({
projectRoots: opts.projectRoots,
blacklistRE: opts.blacklistRE,
polyfillModuleNames: opts.polyfillModuleNames,
moduleFormat: opts.moduleFormat,
assetRoots: opts.assetRoots,
fileWatcher: opts.fileWatcher,
assetExts: opts.assetExts,
assetRoots: opts.assetRoots,
blacklistRE: opts.blacklistRE,
cache: this._cache,
extraNodeModules: opts.extraNodeModules,
fileWatcher: opts.fileWatcher,
minifyCode: this._transformer.minify,
moduleFormat: opts.moduleFormat,
polyfillModuleNames: opts.polyfillModuleNames,
projectRoots: opts.projectRoots,
resetCache: opts.resetCache,
transformCode:
(module, code, options) =>
this._transformer.transformFile(module.path, code, options),
extraNodeModules: opts.extraNodeModules,
minifyCode: this._transformer.minify,
});
this._projectRoots = opts.projectRoots;

View File

@ -57,6 +57,10 @@ const validateOpts = declareOpts({
minifyCode: {
type: 'function',
},
resetCache: {
type: 'boolean',
default: false,
},
});
const getDependenciesValidateOpts = declareOpts({
@ -109,6 +113,8 @@ class Resolver {
transformCode: opts.transformCode,
extraNodeModules: opts.extraNodeModules,
assetDependencies: ['react-native/Libraries/Image/AssetRegistry'],
// for jest-haste-map
resetCache: options.resetCache,
});
this._minifyCode = opts.minifyCode;

View File

@ -87,6 +87,7 @@ describe('processRequest', () => {
jest.fn().mockReturnValue({
getDependecyGraph: jest.fn().mockReturnValue({
getHasteMap: jest.fn().mockReturnValue({on: jest.fn()}),
load: jest.fn(() => Promise.resolve()),
}),
});

View File

@ -241,14 +241,14 @@ class Server {
this._fileWatcher.on('all', this._onFileChange.bind(this));
// changes to the haste map can affect resolution of files in the bundle
this._bundler
.getResolver()
.getDependecyGraph()
.getHasteMap()
.on('change', () => {
const dependencyGraph = this._bundler.getResolver().getDependecyGraph();
dependencyGraph.load().then(() => {
dependencyGraph.getHasteMap().on('change', () => {
debug('Clearing bundle cache due to haste map change');
this._clearBundles();
});
});
this._debouncedFileChangeHandler = debounceAndBatch(filePaths => {
// only clear bundles for non-JS changes
@ -292,7 +292,7 @@ class Server {
}
buildBundle(options) {
return Promise.resolve().then(() => {
return this._bundler.getResolver().getDependecyGraph().load().then(() => {
if (!options.platform) {
options.platform = getPlatformExtension(options.entryFile);
}
@ -692,7 +692,11 @@ class Server {
}
},
error => this._handleError(res, JSON.stringify(options), error)
).done();
).catch(error => {
process.nextTick(() => {
throw error;
});
});
}
_symbolicate(req, res) {

View File

@ -9,73 +9,22 @@
'use strict';
const AssetModule_DEPRECATED = require('../AssetModule_DEPRECATED');
const Fastfs = require('../fastfs');
const debug = require('debug')('ReactNativePackager:DependencyGraph');
const path = require('../fastpath');
class DeprecatedAssetMap {
constructor({
fsCrawl,
roots,
assetExts,
fileWatcher,
ignoreFilePath,
helpers,
activity,
enabled,
platforms,
files,
}) {
if (roots == null || roots.length === 0 || !enabled) {
this._disabled = true;
return;
}
this._helpers = helpers;
this._map = Object.create(null);
this._assetExts = assetExts;
this._activity = activity;
this._platforms = platforms;
if (!this._disabled) {
this._fastfs = new Fastfs(
'Assets',
roots,
fileWatcher,
{ ignore: ignoreFilePath, crawling: fsCrawl, activity }
);
this._fastfs.on('change', this._processFileChange.bind(this));
}
}
build() {
if (this._disabled) {
return Promise.resolve();
}
return this._fastfs.build().then(
() => {
const activity = this._activity;
let processAsset_DEPRECATEDActivity;
if (activity) {
processAsset_DEPRECATEDActivity = activity.startEvent(
'Building (deprecated) Asset Map',
null,
{
telemetric: true,
},
);
}
this._fastfs.findFilesByExts(this._assetExts).forEach(
file => this._processAsset(file)
);
if (activity) {
activity.endEvent(processAsset_DEPRECATEDActivity);
}
}
);
files.forEach(file => this._processAsset(file));
}
resolve(fromModule, toModuleName) {
@ -105,7 +54,7 @@ class DeprecatedAssetMap {
}
}
_processFileChange(type, filePath, root, fstat) {
processFileChange(type, filePath, root, fstat) {
const name = assetName(filePath);
if (type === 'change' || type === 'delete') {
delete this._map[name];

View File

@ -77,6 +77,15 @@ fs.readFile.mockImpl(function(filepath, encoding, callback) {
}
});
fs.readFileSync.mockImpl(function(filepath, encoding) {
const node = getToNode(filepath);
// dir check
if (node && typeof node === 'object' && node.SYMLINK == null) {
throw new Error('Error readFileSync a dir: ' + filepath);
}
return node;
});
fs.stat.mockImpl((filepath, callback) => {
callback = asyncCallback(callback);
let node;
@ -121,6 +130,31 @@ fs.statSync.mockImpl((filepath) => {
};
});
fs.lstat.mockImpl((filepath, callback) => {
callback = asyncCallback(callback);
let node;
try {
node = getToNode(filepath);
} catch (e) {
callback(e);
return;
}
if (node && typeof node === 'object') {
callback(null, {
isDirectory: () => true,
isSymbolicLink: () => false,
mtime,
});
} else {
callback(null, {
isDirectory: () => false,
isSymbolicLink: () => false,
mtime,
});
}
});
fs.lstatSync.mockImpl((filepath) => {
const node = getToNode(filepath);

View File

@ -12,6 +12,99 @@ jest.autoMockOff();
jest.useRealTimers();
jest.mock('fs');
// This is an ugly hack:
// * jest-haste-map uses `find` for fast file system crawling which won't work
// when we mock the file system in node. This mock copies the node crawler's
// implementation and always falls back to the node crawling mechanism.
// Ideally we'll make this an option in jest-haste-map to force it to use
// the node crawler.
jest.mock('jest-haste-map/build/crawlers/node', () => {
const H = require('jest-haste-map/build/constants');
const fs = require('fs');
const path = require('path');
function find(
roots,
extensions,
ignore,
callback)
{
const result = [];
let activeCalls = 0;
function search(directory) {
activeCalls++;
fs.readdir(directory, (err, names) => {
activeCalls--;
names.forEach(file => {
file = process.platform === 'win32' ?
path.win32.join(directory, file) :
path.join(directory, file);
if (ignore(file)) {
return;
}
activeCalls++;
fs.lstat(file, (err, stat) => {
activeCalls--;
if (!err && stat && !stat.isSymbolicLink()) {
if (stat.isDirectory()) {
search(file);
} else {
const ext = path.extname(file).substr(1);
if (extensions.indexOf(ext) !== -1) {
result.push([file, stat.mtime.getTime()]);
}
}
}
if (activeCalls === 0) {
callback(result);
}
});
});
if (activeCalls === 0) {
callback(result);
}
});
}
roots.forEach(search);
}
return function nodeCrawl(
roots,
extensions,
ignore,
data)
{
return new Promise(resolve => {
const callback = list => {
const files = Object.create(null);
list.forEach(fileData => {
const name = fileData[0];
const mtime = fileData[1];
const existingFile = data.files[name];
if (existingFile && existingFile[H.MTIME] === mtime) {
files[name] = existingFile;
} else {
// See ../constants.js
files[name] = ['', mtime, 0, []];
}
});
data.files = files;
resolve(data);
};
find(roots, extensions, ignore, callback);
});
};
});
const mocksPattern = /(?:[\\/]|^)__mocks__[\\/]([^\/]+)\.js$/;
jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000;
@ -103,11 +196,15 @@ describe('DependencyGraph', function() {
],
platforms: ['ios', 'android'],
shouldThrowOnUnresolvedErrors: () => false,
useWatchman: false,
maxWorkers: 1,
resetCache: true,
};
});
describe('get sync dependencies (posix)', function() {
let DependencyGraph;
const consoleWarn = console.warn;
const realPlatform = process.platform;
beforeEach(function() {
process.platform = 'linux';
@ -115,10 +212,11 @@ describe('DependencyGraph', function() {
});
afterEach(function() {
console.warn = consoleWarn;
process.platform = realPlatform;
});
pit('should get dependencies', function() {
it('should get dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -186,7 +284,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve relative entry path', function() {
it('should resolve relative entry path', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -219,7 +317,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get shallow dependencies', function() {
it('should get shallow dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -274,7 +372,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with the correct extensions', function() {
it('should get dependencies with the correct extensions', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -328,7 +426,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get json dependencies', function() {
it('should get json dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -388,7 +486,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get package json as a dep', () => {
it('should get package json as a dep', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -435,7 +533,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with deprecated assets', function() {
it('should get dependencies with deprecated assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -483,7 +581,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with relative assets', function() {
it('should get dependencies with relative assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -533,7 +631,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with assets and resolution', function() {
it('should get dependencies with assets and resolution', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -612,7 +710,7 @@ describe('DependencyGraph', function() {
});
});
pit('should respect platform extension in assets', function() {
it('should respect platform extension in assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -692,7 +790,7 @@ describe('DependencyGraph', function() {
});
});
pit('Deprecated and relative assets can live together', function() {
it('Deprecated and relative assets can live together', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -754,7 +852,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get recursive dependencies', function() {
it('should get recursive dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -804,7 +902,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages', function() {
it('should work with packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -855,7 +953,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages', function() {
it('should work with packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -906,7 +1004,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with a dot in the name', function() {
it('should work with packages with a dot in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -975,7 +1073,7 @@ describe('DependencyGraph', function() {
});
});
pit('should default main package to index.js', function() {
it('should default main package to index.js', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1020,7 +1118,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve using alternative ids', () => {
it('should resolve using alternative ids', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -1069,7 +1167,7 @@ describe('DependencyGraph', function() {
});
});
pit('should default use index.js if main is a dir', function() {
it('should default use index.js if main is a dir', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1117,7 +1215,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve require to index if it is a dir', function() {
it('should resolve require to index if it is a dir', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1162,7 +1260,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve require to main if it is a dir w/ a package.json', function() {
it('should resolve require to main if it is a dir w/ a package.json', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1211,7 +1309,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore malformed packages', function() {
it('should ignore malformed packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1248,8 +1346,9 @@ describe('DependencyGraph', function() {
});
});
pit('should fatal on multiple modules with the same name', function() {
var root = '/root';
it('should fatal on multiple modules with the same name', function() {
const root = '/root';
console.warn = jest.fn();
setMockFileSystem({
'root': {
'index.js': [
@ -1279,10 +1378,11 @@ describe('DependencyGraph', function() {
'with the same name across two different files.'
);
expect(err.type).toEqual('DependencyGraphError');
expect(console.warn).toBeCalled();
});
});
pit('should be forgiving with missing requires', function() {
it('should be forgiving with missing requires', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1317,7 +1417,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with subdirs', function() {
it('should work with packages with subdirs', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1373,7 +1473,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with symlinked subdirs', function() {
it('should work with packages with symlinked subdirs', function() {
var root = '/root';
setMockFileSystem({
'symlinkedPackage': {
@ -1430,7 +1530,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with relative modules in packages', function() {
it('should work with relative modules in packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1522,7 +1622,7 @@ describe('DependencyGraph', function() {
}
function testBrowserField(fieldName) {
pit('should support simple browser field in packages ("' + fieldName + '")', function() {
it('should support simple browser field in packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1577,7 +1677,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() {
it('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1630,7 +1730,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support mapping main in browser field json ("' + fieldName + '")', function() {
it('should support mapping main in browser field json ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1686,7 +1786,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() {
it('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1744,7 +1844,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping of files ("' + fieldName + '")', function() {
it('should support browser mapping of files ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1848,7 +1948,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping for packages ("' + fieldName + '")', function() {
it('should support browser mapping for packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1920,7 +2020,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping of a package to a file ("' + fieldName + '")', () => {
it('should support browser mapping of a package to a file ("' + fieldName + '")', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -1999,7 +2099,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping for packages ("' + fieldName + '")', function() {
it('should support browser mapping for packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2071,7 +2171,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser exclude of a package ("' + fieldName + '")', function() {
it('should support browser exclude of a package ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2128,7 +2228,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser exclude of a file ("' + fieldName + '")', function() {
it('should support browser exclude of a file ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2181,7 +2281,7 @@ describe('DependencyGraph', function() {
});
}
pit('should fall back to browser mapping from react-native mapping', function() {
it('should fall back to browser mapping from react-native mapping', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2273,7 +2373,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with absolute paths', () => {
it('should work with absolute paths', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2313,7 +2413,7 @@ describe('DependencyGraph', function() {
});
});
pit('should merge browser mapping with react-native mapping', function() {
it('should merge browser mapping with react-native mapping', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2451,7 +2551,7 @@ describe('DependencyGraph', function() {
});
});
pit('should fall back to `extraNodeModules`', () => {
it('should fall back to `extraNodeModules`', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2513,7 +2613,7 @@ describe('DependencyGraph', function() {
});
});
pit(
it(
'should only use `extraNodeModules` after checking all possible filesystem locations',
() => {
const root = '/root';
@ -2561,7 +2661,7 @@ describe('DependencyGraph', function() {
}
);
pit('should be able to resolve paths within `extraNodeModules`', () => {
it('should be able to resolve paths within `extraNodeModules`', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2692,7 +2792,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with absolute paths', () => {
it('should work with absolute paths', () => {
const root = 'C:\\root';
setMockFileSystem({
'root': {
@ -2745,7 +2845,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('should work with nested node_modules', function() {
it('should work with nested node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2835,7 +2935,7 @@ describe('DependencyGraph', function() {
});
});
pit('platform should work with node_modules', function() {
it('platform should work with node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2905,7 +3005,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with specific paths', function() {
it('nested node_modules with specific paths', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2996,7 +3096,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with browser field', function() {
it('nested node_modules with browser field', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3091,7 +3191,7 @@ describe('DependencyGraph', function() {
});
});
pit('node_modules should support multi level', function() {
it('node_modules should support multi level', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3165,7 +3265,7 @@ describe('DependencyGraph', function() {
});
});
pit('should selectively ignore providesModule in node_modules', function() {
it('should selectively ignore providesModule in node_modules', function() {
var root = '/root';
var otherRoot = '/anotherRoot';
setMockFileSystem({
@ -3339,7 +3439,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not be confused by prev occuring whitelisted names', function() {
it('should not be confused by prev occuring whitelisted names', function() {
var root = '/react-haste';
setMockFileSystem({
'react-haste': {
@ -3397,7 +3497,7 @@ describe('DependencyGraph', function() {
});
pit('should ignore modules it cant find (assumes own require system)', function() {
it('should ignore modules it cant find (assumes own require system)', function() {
// For example SourceMap.js implements it's own require system.
var root = '/root';
setMockFileSystem({
@ -3441,7 +3541,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with node packages with a .js in the name', function() {
it('should work with node packages with a .js in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3494,7 +3594,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (haste)', function() {
it('should work with multiple platforms (haste)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3553,7 +3653,7 @@ describe('DependencyGraph', function() {
});
});
pit('should pick the generic file', function() {
it('should pick the generic file', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3613,7 +3713,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (node)', function() {
it('should work with multiple platforms (node)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3660,7 +3760,7 @@ describe('DependencyGraph', function() {
});
});
pit('should require package.json', () => {
it('should require package.json', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -3741,7 +3841,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with one-character node_modules', () => {
it('should work with one-character node_modules', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -3796,7 +3896,7 @@ describe('DependencyGraph', function() {
const DependencyGraph = require('../index');
pit('should work with nested node_modules', function() {
it('should work with nested node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3886,7 +3986,7 @@ describe('DependencyGraph', function() {
});
});
pit('platform should work with node_modules', function() {
it('platform should work with node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3956,7 +4056,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with specific paths', function() {
it('nested node_modules with specific paths', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4047,7 +4147,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with browser field', function() {
it('nested node_modules with browser field', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4142,7 +4242,7 @@ describe('DependencyGraph', function() {
});
});
pit('node_modules should support multi level', function() {
it('node_modules should support multi level', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4216,7 +4316,7 @@ describe('DependencyGraph', function() {
});
});
pit('should selectively ignore providesModule in node_modules', function() {
it('should selectively ignore providesModule in node_modules', function() {
var root = '/root';
var otherRoot = '/anotherRoot';
setMockFileSystem({
@ -4390,7 +4490,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not be confused by prev occuring whitelisted names', function() {
it('should not be confused by prev occuring whitelisted names', function() {
var root = '/react-haste';
setMockFileSystem({
'react-haste': {
@ -4447,7 +4547,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore modules it cant find (assumes own require system)', function() {
it('should ignore modules it cant find (assumes own require system)', function() {
// For example SourceMap.js implements it's own require system.
var root = '/root';
setMockFileSystem({
@ -4491,7 +4591,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with node packages with a .js in the name', function() {
it('should work with node packages with a .js in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4544,7 +4644,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (haste)', function() {
it('should work with multiple platforms (haste)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4603,7 +4703,7 @@ describe('DependencyGraph', function() {
});
});
pit('should pick the generic file', function() {
it('should pick the generic file', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4662,7 +4762,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (node)', function() {
it('should work with multiple platforms (node)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4709,7 +4809,7 @@ describe('DependencyGraph', function() {
});
});
pit('should require package.json', () => {
it('should require package.json', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -4824,7 +4924,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('updates module dependencies', function() {
it('updates module dependencies', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -4887,7 +4987,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file change', function() {
it('updates module dependencies on file change', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -4950,7 +5050,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file delete', function() {
it('updates module dependencies on file delete', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5012,7 +5112,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file add', function() {
it('updates module dependencies on file add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5105,7 +5205,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on deprecated asset add', function() {
it('updates module dependencies on deprecated asset add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5174,7 +5274,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on relative asset add', function() {
it('updates module dependencies on relative asset add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5244,7 +5344,7 @@ describe('DependencyGraph', function() {
});
});
pit('runs changes through ignore filter', function() {
it('runs changes through ignore filter', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5334,7 +5434,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore directory updates', function() {
it('should ignore directory updates', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -5409,7 +5509,7 @@ describe('DependencyGraph', function() {
});
});
pit('changes to browser field', function() {
it('changes to browser field', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5472,7 +5572,7 @@ describe('DependencyGraph', function() {
});
});
pit('removes old package from cache', function() {
it('removes old package from cache', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5523,7 +5623,7 @@ describe('DependencyGraph', function() {
});
});
pit('should update node package changes', function() {
it('should update node package changes', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5629,7 +5729,7 @@ describe('DependencyGraph', function() {
});
});
pit('should update node package main changes', function() {
it('should update node package main changes', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5694,7 +5794,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not error when the watcher reports a known file as added', function() {
it('should not error when the watcher reports a known file as added', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -5737,7 +5837,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('supports custom file extensions', () => {
it('supports custom file extensions', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5808,7 +5908,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('resolves to null if mocksPattern is not specified', () => {
it('resolves to null if mocksPattern is not specified', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5830,7 +5930,7 @@ describe('DependencyGraph', function() {
});
});
pit('retrieves a list of all required mocks', () => {
it('retrieves a list of all required mocks', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5863,7 +5963,7 @@ describe('DependencyGraph', function() {
});
});
pit('adds mocks as a dependency of their actual module', () => {
it('adds mocks as a dependency of their actual module', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5936,7 +6036,7 @@ describe('DependencyGraph', function() {
});
});
pit('resolves mocks that do not have a real module associated with them', () => {
it('resolves mocks that do not have a real module associated with them', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -6035,20 +6135,20 @@ describe('DependencyGraph', function() {
});
});
pit('calls back for each finished module', () => {
it('calls back for each finished module', () => {
return getDependencies().then(() =>
expect(onProgress.mock.calls.length).toBe(8)
);
});
pit('increases the number of finished modules in steps of one', () => {
it('increases the number of finished modules in steps of one', () => {
return getDependencies().then(() => {
const increments = onProgress.mock.calls.map(([finished]) => finished);
expect(increments).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
});
});
pit('adds the number of discovered modules to the number of total modules', () => {
it('adds the number of discovered modules to the number of total modules', () => {
return getDependencies().then(() => {
const increments = onProgress.mock.calls.map(([, total]) => total);
expect(increments).toEqual([3, 5, 6, 6, 7, 7, 8, 8]);
@ -6062,7 +6162,7 @@ describe('DependencyGraph', function() {
DependencyGraph = require('../index');
});
pit('allows setting dependencies for asset modules', () => {
it('allows setting dependencies for asset modules', () => {
const assetDependencies = ['arbitrary', 'dependencies'];
setMockFileSystem({
@ -6121,7 +6221,7 @@ describe('DependencyGraph', function() {
roots: ['/root'],
});
moduleReadDeferreds = {};
callDeferreds = [defer()/* a.js */, defer()/* b.js */];
callDeferreds = [defer(), defer()]; // [a.js, b.js]
Module.prototype.read = jest.genMockFn().mockImplementation(function() {
const returnValue = moduleRead.apply(this, arguments);
@ -6143,7 +6243,7 @@ describe('DependencyGraph', function() {
Module.prototype.read = moduleRead;
});
pit('produces a deterministic tree if the "a" module resolves first', () => {
it('produces a deterministic tree if the "a" module resolves first', () => {
const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js');
return Promise.all(callDeferreds.map(deferred => deferred.promise))
@ -6168,7 +6268,7 @@ describe('DependencyGraph', function() {
});
});
pit('produces a deterministic tree if the "b" module resolves first', () => {
it('produces a deterministic tree if the "b" module resolves first', () => {
const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js');
return Promise.all(callDeferreds.map(deferred => deferred.promise))

View File

@ -78,17 +78,16 @@ describe('Module', () => {
const createJSONModule =
(options) => createModule({...options, file: '/root/package.json'});
beforeEach(function(done) {
beforeEach(function() {
process.platform = 'linux';
cache = createCache();
fastfs = new Fastfs(
'test',
['/root'],
fileWatcher,
{crawling: Promise.resolve([fileName, '/root/package.json']), ignore: []},
['/root/index.js', '/root/package.json'],
{ignore: []},
);
fastfs.build().then(done);
});
describe('Module ID', () => {

View File

@ -22,13 +22,17 @@ const contents = fs.readFileSync(fileName, 'utf-8');
describe('fastfs:', function() {
let fastfs;
const crawling = Promise.resolve([fileName]);
const roots = [__dirname];
const watcher = new EventEmitter();
beforeEach(function(done) {
fastfs = new Fastfs('arbitrary', roots, watcher, {crawling});
fastfs.build().then(done);
beforeEach(function() {
fastfs = new Fastfs(
'arbitrary',
roots,
watcher,
[`${__dirname}/fastfs-data`],
{}
);
});
describe('partial reading', () => {

View File

@ -1,13 +0,0 @@
'use strict';
const nodeCrawl = require('./node');
const watchmanCrawl = require('./watchman');
function crawl(roots, options) {
const {fileWatcher} = options;
return (fileWatcher ? fileWatcher.isWatchman() : Promise.resolve(false)).then(
isWatchman => isWatchman ? watchmanCrawl(roots, options) : nodeCrawl(roots, options)
);
}
module.exports = crawl;

View File

@ -1,61 +0,0 @@
'use strict';
const denodeify = require('denodeify');
const debug = require('debug')('ReactNativePackager:DependencyGraph');
const fs = require('graceful-fs');
const path = require('../fastpath');
const readDir = denodeify(fs.readdir);
const stat = denodeify(fs.stat);
function nodeRecReadDir(roots, {ignore, exts}) {
const queue = roots.slice();
const retFiles = [];
const extPattern = new RegExp(
'\.(' + exts.join('|') + ')$'
);
function search() {
const currDir = queue.shift();
if (!currDir) {
return Promise.resolve();
}
return readDir(currDir)
.then(files => files.map(f => path.join(currDir, f)))
.then(files => Promise.all(
files.map(f => stat(f).catch(handleBrokenLink))
).then(stats => [
// Remove broken links.
files.filter((file, i) => !!stats[i]),
stats.filter(Boolean),
]))
.then(([files, stats]) => {
files.forEach((filePath, i) => {
if (ignore(filePath)) {
return;
}
if (stats[i].isDirectory()) {
queue.push(filePath);
return;
}
if (filePath.match(extPattern)) {
retFiles.push(path.resolve(filePath));
}
});
return search();
});
}
return search().then(() => retFiles);
}
function handleBrokenLink(e) {
debug('WARNING: error stating, possibly broken symlink', e.message);
return Promise.resolve();
}
module.exports = nodeRecReadDir;

View File

@ -1,76 +0,0 @@
'use strict';
const denodeify = require('denodeify');
const path = require('../fastpath');
const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html';
function watchmanRecReadDir(roots, {ignore, fileWatcher, exts}) {
const files = [];
return Promise.all(
roots.map(
root => fileWatcher.getWatcherForRoot(root)
)
).then(
watchers => {
// All watchman roots for all watches we have.
const watchmanRoots = watchers.map(
watcher => watcher.watchProjectInfo.root
);
// Actual unique watchers (because we use watch-project we may end up with
// duplicate "real" watches, and that's by design).
// TODO(amasad): push this functionality into the `FileWatcher`.
const uniqueWatchers = watchers.filter(
(watcher, i) => watchmanRoots.indexOf(watcher.watchProjectInfo.root) === i
);
return Promise.all(
uniqueWatchers.map(watcher => {
const watchedRoot = watcher.watchProjectInfo.root;
// Build up an expression to filter the output by the relevant roots.
const dirExpr = ['anyof'];
for (let i = 0; i < roots.length; i++) {
const root = roots[i];
if (isDescendant(watchedRoot, root)) {
dirExpr.push(['dirname', path.relative(watchedRoot, root)]);
}
}
const cmd = denodeify(watcher.client.command.bind(watcher.client));
return cmd(['query', watchedRoot, {
suffix: exts,
expression: ['allof', ['type', 'f'], 'exists', dirExpr],
fields: ['name'],
}]).then(resp => {
if ('warning' in resp) {
console.warn('watchman warning: ', resp.warning);
}
resp.files.forEach(filePath => {
filePath = watchedRoot + path.sep + filePath;
if (!ignore(filePath)) {
files.push(filePath);
}
return false;
});
});
})
);
}).then(
() => files,
error => {
throw new Error(
`Watchman error: ${error.message.trim()}. Make sure watchman ` +
`is running for this project. See ${watchmanURL}.`
);
}
);
}
function isDescendant(root, child) {
return root === child || child.startsWith(root + path.sep);
}
module.exports = watchmanRecReadDir;

View File

@ -20,7 +20,7 @@ const stat = denodeify(fs.stat);
const NOT_FOUND_IN_ROOTS = 'NotFoundInRootsError';
class Fastfs extends EventEmitter {
constructor(name, roots, fileWatcher, {ignore, crawling, activity}) {
constructor(name, roots, fileWatcher, files, {ignore, activity}) {
super();
this._name = name;
this._fileWatcher = fileWatcher;
@ -37,45 +37,39 @@ class Fastfs extends EventEmitter {
return new File(root, true);
});
this._fastPaths = Object.create(null);
this._crawling = crawling;
this._activity = activity;
}
build() {
return this._crawling.then(files => {
let fastfsActivity;
const activity = this._activity;
if (activity) {
fastfsActivity = activity.startEvent(
'Building in-memory fs for ' + this._name,
null,
{
telemetric: true,
},
);
}
files.forEach(filePath => {
const root = this._getRoot(filePath);
if (root) {
const newFile = new File(filePath, false);
const dirname = filePath.substr(0, filePath.lastIndexOf(path.sep));
const parent = this._fastPaths[dirname];
this._fastPaths[filePath] = newFile;
if (parent) {
parent.addChild(newFile, this._fastPaths);
} else {
root.addChild(newFile, this._fastPaths);
}
let fastfsActivity;
if (activity) {
fastfsActivity = activity.startEvent(
'Building in-memory fs for ' + this._name,
null,
{
telemetric: true,
},
);
}
files.forEach(filePath => {
const root = this._getRoot(filePath);
if (root) {
const newFile = new File(filePath, false);
const dirname = filePath.substr(0, filePath.lastIndexOf(path.sep));
const parent = this._fastPaths[dirname];
this._fastPaths[filePath] = newFile;
if (parent) {
parent.addChild(newFile, this._fastPaths);
} else {
root.addChild(newFile, this._fastPaths);
}
});
if (activity) {
activity.endEvent(fastfsActivity);
}
if (this._fileWatcher) {
this._fileWatcher.on('all', this._processFileChange.bind(this));
}
});
if (activity) {
activity.endEvent(fastfsActivity);
}
if (this._fileWatcher) {
this._fileWatcher.on('all', this._processFileChange.bind(this));
}
}
stat(filePath) {

View File

@ -11,10 +11,10 @@
const Cache = require('./Cache');
const Fastfs = require('./fastfs');
const FileWatcher = require('./FileWatcher');
const JestHasteMap = require('jest-haste-map');
const Module = require('./Module');
const ModuleCache = require('./ModuleCache');
const Polyfill = require('./Polyfill');
const crawl = require('./crawlers');
const extractRequires = require('./lib/extractRequires');
const getAssetDataFromName = require('./lib/getAssetDataFromName');
const getInverseDependencies = require('./lib/getInverseDependencies');
@ -23,6 +23,7 @@ const isAbsolutePath = require('absolute-path');
const replacePatterns = require('./lib/replacePatterns');
const path = require('./fastpath');
const util = require('util');
const os = require('os');
const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers');
const ResolutionRequest = require('./DependencyGraph/ResolutionRequest');
const ResolutionResponse = require('./DependencyGraph/ResolutionResponse');
@ -57,6 +58,10 @@ class DependencyGraph {
assetDependencies,
moduleOptions,
extraNodeModules,
// additional arguments for jest-haste-map
useWatchman,
maxWorkers,
resetCache,
}) {
this._opts = {
activity: activity || defaultActivity,
@ -78,6 +83,10 @@ class DependencyGraph {
cacheTransformResults: true,
},
extraNodeModules,
// additional arguments for jest-haste-map & defaults
useWatchman: useWatchman !== false,
maxWorkers,
resetCache,
};
this._cache = cache;
this._assetDependencies = assetDependencies;
@ -90,103 +99,110 @@ class DependencyGraph {
return this._loading;
}
const {activity} = this._opts;
const depGraphActivity = activity.startEvent(
'Initializing Packager',
null,
{
telemetric: true,
},
);
const crawlActivity = activity.startEvent(
'Crawling File System',
null,
{
telemetric: true,
},
);
const allRoots = this._opts.roots.concat(this._opts.assetRoots_DEPRECATED);
this._crawling = crawl(allRoots, {
ignore: this._opts.ignoreFilePath,
exts: this._opts.extensions.concat(this._opts.assetExts),
fileWatcher: this._opts.fileWatcher,
});
this._crawling.then((files) => activity.endEvent(crawlActivity));
this._fastfs = new Fastfs(
'JavaScript',
this._opts.roots,
this._opts.fileWatcher,
{
ignore: this._opts.ignoreFilePath,
crawling: this._crawling,
activity: activity,
}
);
this._fastfs.on('change', this._processFileChange.bind(this));
this._moduleCache = new ModuleCache({
fastfs: this._fastfs,
cache: this._cache,
extractRequires: this._opts.extractRequires,
transformCode: this._opts.transformCode,
depGraphHelpers: this._helpers,
assetDependencies: this._assetDependencies,
moduleOptions: this._opts.moduleOptions,
}, this._opts.platforms);
this._hasteMap = new HasteMap({
fastfs: this._fastfs,
extensions: this._opts.extensions,
moduleCache: this._moduleCache,
preferNativePlatform: this._opts.preferNativePlatform,
helpers: this._helpers,
platforms: this._opts.platforms,
const mw = this._opts.maxWorkers;
const haste = new JestHasteMap({
extensions: this._opts.extensions.concat(this._opts.assetExts),
ignorePattern: {test: this._opts.ignoreFilePath},
maxWorkers: typeof mw === 'number' && mw >= 1 ? mw : getMaxWorkers(),
mocksPattern: '',
name: 'react-native-packager',
platforms: Array.from(this._opts.platforms),
providesModuleNodeModules: this._opts.providesModuleNodeModules,
resetCache: this._opts.resetCache,
retainAllFiles: true,
roots: this._opts.roots.concat(this._opts.assetRoots_DEPRECATED),
useWatchman: this._opts.useWatchman,
});
this._deprecatedAssetMap = new DeprecatedAssetMap({
fsCrawl: this._crawling,
roots: this._opts.assetRoots_DEPRECATED,
helpers: this._helpers,
fileWatcher: this._opts.fileWatcher,
ignoreFilePath: this._opts.ignoreFilePath,
assetExts: this._opts.assetExts,
activity: this._opts.activity,
enabled: this._opts.enableAssetMap,
platforms: this._opts.platforms,
});
this._loading = haste.build().then(hasteMap => {
const {activity} = this._opts;
const depGraphActivity = activity.startEvent(
'Initializing Packager',
null,
{
telemetric: true,
},
);
this._loading = Promise.all([
this._fastfs.build()
.then(() => {
const hasteActivity = activity.startEvent(
'Building Haste Map',
null,
{
telemetric: true,
},
const hasteFSFiles = hasteMap.hasteFS.getAllFiles();
this._fastfs = new Fastfs(
'JavaScript',
this._opts.roots,
this._opts.fileWatcher,
hasteFSFiles,
{
ignore: this._opts.ignoreFilePath,
activity: activity,
}
);
this._fastfs.on('change', this._processFileChange.bind(this));
this._moduleCache = new ModuleCache({
fastfs: this._fastfs,
cache: this._cache,
extractRequires: this._opts.extractRequires,
transformCode: this._opts.transformCode,
depGraphHelpers: this._helpers,
assetDependencies: this._assetDependencies,
moduleOptions: this._opts.moduleOptions,
}, this._opts.platforms);
this._hasteMap = new HasteMap({
fastfs: this._fastfs,
extensions: this._opts.extensions,
moduleCache: this._moduleCache,
preferNativePlatform: this._opts.preferNativePlatform,
helpers: this._helpers,
platforms: this._opts.platforms,
});
const escapePath = (p: string) => {
return (path.sep === '\\') ? p.replace(/(\/|\\(?!\.))/g, '\\\\') : p;
};
const assetPattern =
new RegExp('^' + this._opts.assetRoots_DEPRECATED.map(escapePath).join('|'));
const assetFiles = hasteMap.hasteFS.matchFiles(assetPattern);
this._deprecatedAssetMap = new DeprecatedAssetMap({
helpers: this._helpers,
assetExts: this._opts.assetExts,
platforms: this._opts.platforms,
files: assetFiles,
});
this._fastfs.on('change', (type, filePath, root, fstat) => {
if (assetPattern.test(path.join(root, filePath))) {
this._deprecatedAssetMap.processFileChange(type, filePath, root, fstat);
}
});
const hasteActivity = activity.startEvent(
'Building Haste Map',
null,
{
telemetric: true,
},
);
return this._hasteMap.build().then(
map => {
activity.endEvent(hasteActivity);
activity.endEvent(depGraphActivity);
return map;
},
err => {
const error = new Error(
`Failed to build DependencyGraph: ${err.message}`
);
return this._hasteMap.build().then(map => {
activity.endEvent(hasteActivity);
return map;
});
}),
this._deprecatedAssetMap.build(),
]).then(
response => {
activity.endEvent(depGraphActivity);
return response[0]; // Return the haste map
},
err => {
const error = new Error(
`Failed to build DependencyGraph: ${err.message}`
);
error.type = ERROR_BUILDING_DEP_GRAPH;
error.stack = err.stack;
throw error;
}
);
error.type = ERROR_BUILDING_DEP_GRAPH;
error.stack = err.stack;
throw error;
}
);
});
return this._loading;
}
@ -312,7 +328,7 @@ class DependencyGraph {
this._loading = this._hasteMap.build();
} else {
this._loading = this._hasteMap.processFileChange(type, absPath);
this._loading.catch((e) => this._hasteMapError = e);
this._loading.catch((e) => {this._hasteMapError = e;});
}
return this._loading;
};
@ -351,4 +367,28 @@ function NotFoundError() {
}
util.inherits(NotFoundError, Error);
function getMaxWorkers() {
const cores = os.cpus().length;
if (cores <= 1) {
// oh well...
return 1;
}
if (cores <= 4) {
// don't starve the CPU while still reading reasonably rapidly
return cores - 1;
}
if (cores <= 8) {
// empirical testing showed massive diminishing returns when going over
// 4 or 5 workers on 8-core machines
return Math.floor(cores * 0.75) - 1;
}
// pretty much guesswork
if (cores < 24) {
return Math.floor(3 / 8 * cores + 3);
}
return cores / 2;
}
module.exports = DependencyGraph;