status-im
/
react-native
mirror of https://github.com/status-im/react-native.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Replacing node-haste with jest-haste-map 

Summary: Modified `node-haste` implementation to use the much faster `jest-haste-map` under the hood. The underlying `fastfs` now gets passed the entire file list from the `jest-haste-map` rather than crawl the filesystem.

Reviewed By: cpojer

Differential Revision: D3724387

fbshipit-source-id: 447d58ea0edf283662ec23d1e2deee992cf8d240
This commit is contained in:
Ovidiu Viorel Iepure 2016-10-03 05:16:04 -07:00 committed by Facebook Github Bot
parent d7d89172c2
commit 6130650d93
15 changed files with 436 additions and 453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -166,6 +166,7 @@
"image-size": "^0.3.5",
"immutable": "~3.7.6",
"inquirer": "^0.12.0",
"jest-haste-map": "15.0.1",
"joi": "^6.6.1",
"json-stable-stringify": "^1.0.1",
"json5": "^0.4.0",

17
packager/react-packager/src/Bundler/index.js vendored
View File

@ -143,19 +143,20 @@ class Bundler {
});
this._resolver = new Resolver({
projectRoots: opts.projectRoots,
blacklistRE: opts.blacklistRE,
polyfillModuleNames: opts.polyfillModuleNames,
moduleFormat: opts.moduleFormat,
assetRoots: opts.assetRoots,
fileWatcher: opts.fileWatcher,
assetExts: opts.assetExts,
assetRoots: opts.assetRoots,
blacklistRE: opts.blacklistRE,
cache: this._cache,
extraNodeModules: opts.extraNodeModules,
fileWatcher: opts.fileWatcher,
minifyCode: this._transformer.minify,
moduleFormat: opts.moduleFormat,
polyfillModuleNames: opts.polyfillModuleNames,
projectRoots: opts.projectRoots,
resetCache: opts.resetCache,
transformCode:
(module, code, options) =>
this._transformer.transformFile(module.path, code, options),
extraNodeModules: opts.extraNodeModules,
minifyCode: this._transformer.minify,
});
this._projectRoots = opts.projectRoots;

6
packager/react-packager/src/Resolver/index.js vendored
View File

@ -57,6 +57,10 @@ const validateOpts = declareOpts({
minifyCode: {
type: 'function',
},
resetCache: {
type: 'boolean',
default: false,
},
});
const getDependenciesValidateOpts = declareOpts({
@ -109,6 +113,8 @@ class Resolver {
transformCode: opts.transformCode,
extraNodeModules: opts.extraNodeModules,
assetDependencies: ['react-native/Libraries/Image/AssetRegistry'],
// for jest-haste-map
resetCache: options.resetCache,
});
this._minifyCode = opts.minifyCode;

1
packager/react-packager/src/Server/__tests__/Server-test.js vendored
View File

@ -87,6 +87,7 @@ describe('processRequest', () => {
jest.fn().mockReturnValue({
getDependecyGraph: jest.fn().mockReturnValue({
getHasteMap: jest.fn().mockReturnValue({on: jest.fn()}),
load: jest.fn(() => Promise.resolve()),
}),
});

18
packager/react-packager/src/Server/index.js vendored
View File

@ -241,14 +241,14 @@ class Server {
this._fileWatcher.on('all', this._onFileChange.bind(this));
// changes to the haste map can affect resolution of files in the bundle
this._bundler
.getResolver()
.getDependecyGraph()
.getHasteMap()
.on('change', () => {
const dependencyGraph = this._bundler.getResolver().getDependecyGraph();
dependencyGraph.load().then(() => {
dependencyGraph.getHasteMap().on('change', () => {
debug('Clearing bundle cache due to haste map change');
this._clearBundles();
});
});
this._debouncedFileChangeHandler = debounceAndBatch(filePaths => {
// only clear bundles for non-JS changes
@ -292,7 +292,7 @@ class Server {
}
buildBundle(options) {
return Promise.resolve().then(() => {
return this._bundler.getResolver().getDependecyGraph().load().then(() => {
if (!options.platform) {
options.platform = getPlatformExtension(options.entryFile);
}
@ -692,7 +692,11 @@ class Server {
}
},
error => this._handleError(res, JSON.stringify(options), error)
).done();
).catch(error => {
process.nextTick(() => {
throw error;
});
});
}
_symbolicate(req, res) {

57
packager/react-packager/src/node-haste/DependencyGraph/DeprecatedAssetMap.js vendored
View File

@ -9,73 +9,22 @@
'use strict';
const AssetModule_DEPRECATED = require('../AssetModule_DEPRECATED');
const Fastfs = require('../fastfs');
const debug = require('debug')('ReactNativePackager:DependencyGraph');
const path = require('../fastpath');
class DeprecatedAssetMap {
constructor({
fsCrawl,
roots,
assetExts,
fileWatcher,
ignoreFilePath,
helpers,
activity,
enabled,
platforms,
files,
}) {
if (roots == null || roots.length === 0 || !enabled) {
this._disabled = true;
return;
}
this._helpers = helpers;
this._map = Object.create(null);
this._assetExts = assetExts;
this._activity = activity;
this._platforms = platforms;
if (!this._disabled) {
this._fastfs = new Fastfs(
'Assets',
roots,
fileWatcher,
{ ignore: ignoreFilePath, crawling: fsCrawl, activity }
);
this._fastfs.on('change', this._processFileChange.bind(this));
}
}
build() {
if (this._disabled) {
return Promise.resolve();
}
return this._fastfs.build().then(
() => {
const activity = this._activity;
let processAsset_DEPRECATEDActivity;
if (activity) {
processAsset_DEPRECATEDActivity = activity.startEvent(
'Building (deprecated) Asset Map',
null,
{
telemetric: true,
},
);
}
this._fastfs.findFilesByExts(this._assetExts).forEach(
file => this._processAsset(file)
);
if (activity) {
activity.endEvent(processAsset_DEPRECATEDActivity);
}
}
);
files.forEach(file => this._processAsset(file));
}
resolve(fromModule, toModuleName) {
@ -105,7 +54,7 @@ class DeprecatedAssetMap {
}
}
_processFileChange(type, filePath, root, fstat) {
processFileChange(type, filePath, root, fstat) {
const name = assetName(filePath);
if (type === 'change' || type === 'delete') {
delete this._map[name];

34
packager/react-packager/src/node-haste/__mocks__/graceful-fs.js vendored
View File

@ -77,6 +77,15 @@ fs.readFile.mockImpl(function(filepath, encoding, callback) {
}
});
fs.readFileSync.mockImpl(function(filepath, encoding) {
const node = getToNode(filepath);
// dir check
if (node && typeof node === 'object' && node.SYMLINK == null) {
throw new Error('Error readFileSync a dir: ' + filepath);
}
return node;
});
fs.stat.mockImpl((filepath, callback) => {
callback = asyncCallback(callback);
let node;
@ -121,6 +130,31 @@ fs.statSync.mockImpl((filepath) => {
};
});
fs.lstat.mockImpl((filepath, callback) => {
callback = asyncCallback(callback);
let node;
try {
node = getToNode(filepath);
} catch (e) {
callback(e);
return;
}
if (node && typeof node === 'object') {
callback(null, {
isDirectory: () => true,
isSymbolicLink: () => false,
mtime,
});
} else {
callback(null, {
isDirectory: () => false,
isSymbolicLink: () => false,
mtime,
});
}
});
fs.lstatSync.mockImpl((filepath) => {
const node = getToNode(filepath);

292
packager/react-packager/src/node-haste/__tests__/DependencyGraph-test.js vendored
View File

@ -12,6 +12,99 @@ jest.autoMockOff();
jest.useRealTimers();
jest.mock('fs');
// This is an ugly hack:
// * jest-haste-map uses `find` for fast file system crawling which won't work
// when we mock the file system in node. This mock copies the node crawler's
// implementation and always falls back to the node crawling mechanism.
// Ideally we'll make this an option in jest-haste-map to force it to use
// the node crawler.
jest.mock('jest-haste-map/build/crawlers/node', () => {
const H = require('jest-haste-map/build/constants');
const fs = require('fs');
const path = require('path');
function find(
roots,
extensions,
ignore,
callback)
{
const result = [];
let activeCalls = 0;
function search(directory) {
activeCalls++;
fs.readdir(directory, (err, names) => {
activeCalls--;
names.forEach(file => {
file = process.platform === 'win32' ?
path.win32.join(directory, file) :
path.join(directory, file);
if (ignore(file)) {
return;
}
activeCalls++;
fs.lstat(file, (err, stat) => {
activeCalls--;
if (!err && stat && !stat.isSymbolicLink()) {
if (stat.isDirectory()) {
search(file);
} else {
const ext = path.extname(file).substr(1);
if (extensions.indexOf(ext) !== -1) {
result.push([file, stat.mtime.getTime()]);
}
}
}
if (activeCalls === 0) {
callback(result);
}
});
});
if (activeCalls === 0) {
callback(result);
}
});
}
roots.forEach(search);
}
return function nodeCrawl(
roots,
extensions,
ignore,
data)
{
return new Promise(resolve => {
const callback = list => {
const files = Object.create(null);
list.forEach(fileData => {
const name = fileData[0];
const mtime = fileData[1];
const existingFile = data.files[name];
if (existingFile && existingFile[H.MTIME] === mtime) {
files[name] = existingFile;
} else {
// See ../constants.js
files[name] = ['', mtime, 0, []];
}
});
data.files = files;
resolve(data);
};
find(roots, extensions, ignore, callback);
});
};
});
const mocksPattern = /(?:[\\/]|^)__mocks__[\\/]([^\/]+)\.js$/;
jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000;
@ -103,11 +196,15 @@ describe('DependencyGraph', function() {
],
platforms: ['ios', 'android'],
shouldThrowOnUnresolvedErrors: () => false,
useWatchman: false,
maxWorkers: 1,
resetCache: true,
};
});
describe('get sync dependencies (posix)', function() {
let DependencyGraph;
const consoleWarn = console.warn;
const realPlatform = process.platform;
beforeEach(function() {
process.platform = 'linux';
@ -115,10 +212,11 @@ describe('DependencyGraph', function() {
});
afterEach(function() {
console.warn = consoleWarn;
process.platform = realPlatform;
});
pit('should get dependencies', function() {
it('should get dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -186,7 +284,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve relative entry path', function() {
it('should resolve relative entry path', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -219,7 +317,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get shallow dependencies', function() {
it('should get shallow dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -274,7 +372,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with the correct extensions', function() {
it('should get dependencies with the correct extensions', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -328,7 +426,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get json dependencies', function() {
it('should get json dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -388,7 +486,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get package json as a dep', () => {
it('should get package json as a dep', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -435,7 +533,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with deprecated assets', function() {
it('should get dependencies with deprecated assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -483,7 +581,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with relative assets', function() {
it('should get dependencies with relative assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -533,7 +631,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get dependencies with assets and resolution', function() {
it('should get dependencies with assets and resolution', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -612,7 +710,7 @@ describe('DependencyGraph', function() {
});
});
pit('should respect platform extension in assets', function() {
it('should respect platform extension in assets', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -692,7 +790,7 @@ describe('DependencyGraph', function() {
});
});
pit('Deprecated and relative assets can live together', function() {
it('Deprecated and relative assets can live together', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -754,7 +852,7 @@ describe('DependencyGraph', function() {
});
});
pit('should get recursive dependencies', function() {
it('should get recursive dependencies', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -804,7 +902,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages', function() {
it('should work with packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -855,7 +953,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages', function() {
it('should work with packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -906,7 +1004,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with a dot in the name', function() {
it('should work with packages with a dot in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -975,7 +1073,7 @@ describe('DependencyGraph', function() {
});
});
pit('should default main package to index.js', function() {
it('should default main package to index.js', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1020,7 +1118,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve using alternative ids', () => {
it('should resolve using alternative ids', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -1069,7 +1167,7 @@ describe('DependencyGraph', function() {
});
});
pit('should default use index.js if main is a dir', function() {
it('should default use index.js if main is a dir', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1117,7 +1215,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve require to index if it is a dir', function() {
it('should resolve require to index if it is a dir', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1162,7 +1260,7 @@ describe('DependencyGraph', function() {
});
});
pit('should resolve require to main if it is a dir w/ a package.json', function() {
it('should resolve require to main if it is a dir w/ a package.json', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1211,7 +1309,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore malformed packages', function() {
it('should ignore malformed packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1248,8 +1346,9 @@ describe('DependencyGraph', function() {
});
});
pit('should fatal on multiple modules with the same name', function() {
var root = '/root';
it('should fatal on multiple modules with the same name', function() {
const root = '/root';
console.warn = jest.fn();
setMockFileSystem({
'root': {
'index.js': [
@ -1279,10 +1378,11 @@ describe('DependencyGraph', function() {
'with the same name across two different files.'
);
expect(err.type).toEqual('DependencyGraphError');
expect(console.warn).toBeCalled();
});
});
pit('should be forgiving with missing requires', function() {
it('should be forgiving with missing requires', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1317,7 +1417,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with subdirs', function() {
it('should work with packages with subdirs', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1373,7 +1473,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with packages with symlinked subdirs', function() {
it('should work with packages with symlinked subdirs', function() {
var root = '/root';
setMockFileSystem({
'symlinkedPackage': {
@ -1430,7 +1530,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with relative modules in packages', function() {
it('should work with relative modules in packages', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1522,7 +1622,7 @@ describe('DependencyGraph', function() {
}
function testBrowserField(fieldName) {
pit('should support simple browser field in packages ("' + fieldName + '")', function() {
it('should support simple browser field in packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1577,7 +1677,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() {
it('should support browser field in packages w/o .js ext ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1630,7 +1730,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support mapping main in browser field json ("' + fieldName + '")', function() {
it('should support mapping main in browser field json ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1686,7 +1786,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() {
it('should work do correct browser mapping w/o js ext ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1744,7 +1844,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping of files ("' + fieldName + '")', function() {
it('should support browser mapping of files ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1848,7 +1948,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping for packages ("' + fieldName + '")', function() {
it('should support browser mapping for packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -1920,7 +2020,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping of a package to a file ("' + fieldName + '")', () => {
it('should support browser mapping of a package to a file ("' + fieldName + '")', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -1999,7 +2099,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser mapping for packages ("' + fieldName + '")', function() {
it('should support browser mapping for packages ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2071,7 +2171,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser exclude of a package ("' + fieldName + '")', function() {
it('should support browser exclude of a package ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2128,7 +2228,7 @@ describe('DependencyGraph', function() {
});
});
pit('should support browser exclude of a file ("' + fieldName + '")', function() {
it('should support browser exclude of a file ("' + fieldName + '")', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2181,7 +2281,7 @@ describe('DependencyGraph', function() {
});
}
pit('should fall back to browser mapping from react-native mapping', function() {
it('should fall back to browser mapping from react-native mapping', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2273,7 +2373,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with absolute paths', () => {
it('should work with absolute paths', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2313,7 +2413,7 @@ describe('DependencyGraph', function() {
});
});
pit('should merge browser mapping with react-native mapping', function() {
it('should merge browser mapping with react-native mapping', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2451,7 +2551,7 @@ describe('DependencyGraph', function() {
});
});
pit('should fall back to `extraNodeModules`', () => {
it('should fall back to `extraNodeModules`', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2513,7 +2613,7 @@ describe('DependencyGraph', function() {
});
});
pit(
it(
'should only use `extraNodeModules` after checking all possible filesystem locations',
() => {
const root = '/root';
@ -2561,7 +2661,7 @@ describe('DependencyGraph', function() {
}
);
pit('should be able to resolve paths within `extraNodeModules`', () => {
it('should be able to resolve paths within `extraNodeModules`', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -2692,7 +2792,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with absolute paths', () => {
it('should work with absolute paths', () => {
const root = 'C:\\root';
setMockFileSystem({
'root': {
@ -2745,7 +2845,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('should work with nested node_modules', function() {
it('should work with nested node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2835,7 +2935,7 @@ describe('DependencyGraph', function() {
});
});
pit('platform should work with node_modules', function() {
it('platform should work with node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2905,7 +3005,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with specific paths', function() {
it('nested node_modules with specific paths', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -2996,7 +3096,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with browser field', function() {
it('nested node_modules with browser field', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3091,7 +3191,7 @@ describe('DependencyGraph', function() {
});
});
pit('node_modules should support multi level', function() {
it('node_modules should support multi level', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3165,7 +3265,7 @@ describe('DependencyGraph', function() {
});
});
pit('should selectively ignore providesModule in node_modules', function() {
it('should selectively ignore providesModule in node_modules', function() {
var root = '/root';
var otherRoot = '/anotherRoot';
setMockFileSystem({
@ -3339,7 +3439,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not be confused by prev occuring whitelisted names', function() {
it('should not be confused by prev occuring whitelisted names', function() {
var root = '/react-haste';
setMockFileSystem({
'react-haste': {
@ -3397,7 +3497,7 @@ describe('DependencyGraph', function() {
});
pit('should ignore modules it cant find (assumes own require system)', function() {
it('should ignore modules it cant find (assumes own require system)', function() {
// For example SourceMap.js implements it's own require system.
var root = '/root';
setMockFileSystem({
@ -3441,7 +3541,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with node packages with a .js in the name', function() {
it('should work with node packages with a .js in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3494,7 +3594,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (haste)', function() {
it('should work with multiple platforms (haste)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3553,7 +3653,7 @@ describe('DependencyGraph', function() {
});
});
pit('should pick the generic file', function() {
it('should pick the generic file', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3613,7 +3713,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (node)', function() {
it('should work with multiple platforms (node)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3660,7 +3760,7 @@ describe('DependencyGraph', function() {
});
});
pit('should require package.json', () => {
it('should require package.json', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -3741,7 +3841,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with one-character node_modules', () => {
it('should work with one-character node_modules', () => {
const root = '/root';
setMockFileSystem({
[root.slice(1)]: {
@ -3796,7 +3896,7 @@ describe('DependencyGraph', function() {
const DependencyGraph = require('../index');
pit('should work with nested node_modules', function() {
it('should work with nested node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3886,7 +3986,7 @@ describe('DependencyGraph', function() {
});
});
pit('platform should work with node_modules', function() {
it('platform should work with node_modules', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -3956,7 +4056,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with specific paths', function() {
it('nested node_modules with specific paths', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4047,7 +4147,7 @@ describe('DependencyGraph', function() {
});
});
pit('nested node_modules with browser field', function() {
it('nested node_modules with browser field', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4142,7 +4242,7 @@ describe('DependencyGraph', function() {
});
});
pit('node_modules should support multi level', function() {
it('node_modules should support multi level', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4216,7 +4316,7 @@ describe('DependencyGraph', function() {
});
});
pit('should selectively ignore providesModule in node_modules', function() {
it('should selectively ignore providesModule in node_modules', function() {
var root = '/root';
var otherRoot = '/anotherRoot';
setMockFileSystem({
@ -4390,7 +4490,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not be confused by prev occuring whitelisted names', function() {
it('should not be confused by prev occuring whitelisted names', function() {
var root = '/react-haste';
setMockFileSystem({
'react-haste': {
@ -4447,7 +4547,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore modules it cant find (assumes own require system)', function() {
it('should ignore modules it cant find (assumes own require system)', function() {
// For example SourceMap.js implements it's own require system.
var root = '/root';
setMockFileSystem({
@ -4491,7 +4591,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with node packages with a .js in the name', function() {
it('should work with node packages with a .js in the name', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4544,7 +4644,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (haste)', function() {
it('should work with multiple platforms (haste)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4603,7 +4703,7 @@ describe('DependencyGraph', function() {
});
});
pit('should pick the generic file', function() {
it('should pick the generic file', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4662,7 +4762,7 @@ describe('DependencyGraph', function() {
});
});
pit('should work with multiple platforms (node)', function() {
it('should work with multiple platforms (node)', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -4709,7 +4809,7 @@ describe('DependencyGraph', function() {
});
});
pit('should require package.json', () => {
it('should require package.json', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -4824,7 +4924,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('updates module dependencies', function() {
it('updates module dependencies', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -4887,7 +4987,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file change', function() {
it('updates module dependencies on file change', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -4950,7 +5050,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file delete', function() {
it('updates module dependencies on file delete', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5012,7 +5112,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on file add', function() {
it('updates module dependencies on file add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5105,7 +5205,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on deprecated asset add', function() {
it('updates module dependencies on deprecated asset add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5174,7 +5274,7 @@ describe('DependencyGraph', function() {
});
});
pit('updates module dependencies on relative asset add', function() {
it('updates module dependencies on relative asset add', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5244,7 +5344,7 @@ describe('DependencyGraph', function() {
});
});
pit('runs changes through ignore filter', function() {
it('runs changes through ignore filter', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5334,7 +5434,7 @@ describe('DependencyGraph', function() {
});
});
pit('should ignore directory updates', function() {
it('should ignore directory updates', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -5409,7 +5509,7 @@ describe('DependencyGraph', function() {
});
});
pit('changes to browser field', function() {
it('changes to browser field', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5472,7 +5572,7 @@ describe('DependencyGraph', function() {
});
});
pit('removes old package from cache', function() {
it('removes old package from cache', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5523,7 +5623,7 @@ describe('DependencyGraph', function() {
});
});
pit('should update node package changes', function() {
it('should update node package changes', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5629,7 +5729,7 @@ describe('DependencyGraph', function() {
});
});
pit('should update node package main changes', function() {
it('should update node package main changes', function() {
var root = '/root';
var filesystem = setMockFileSystem({
'root': {
@ -5694,7 +5794,7 @@ describe('DependencyGraph', function() {
});
});
pit('should not error when the watcher reports a known file as added', function() {
it('should not error when the watcher reports a known file as added', function() {
var root = '/root';
setMockFileSystem({
'root': {
@ -5737,7 +5837,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('supports custom file extensions', () => {
it('supports custom file extensions', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5808,7 +5908,7 @@ describe('DependencyGraph', function() {
process.platform = realPlatform;
});
pit('resolves to null if mocksPattern is not specified', () => {
it('resolves to null if mocksPattern is not specified', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5830,7 +5930,7 @@ describe('DependencyGraph', function() {
});
});
pit('retrieves a list of all required mocks', () => {
it('retrieves a list of all required mocks', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5863,7 +5963,7 @@ describe('DependencyGraph', function() {
});
});
pit('adds mocks as a dependency of their actual module', () => {
it('adds mocks as a dependency of their actual module', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -5936,7 +6036,7 @@ describe('DependencyGraph', function() {
});
});
pit('resolves mocks that do not have a real module associated with them', () => {
it('resolves mocks that do not have a real module associated with them', () => {
var root = '/root';
setMockFileSystem({
'root': {
@ -6035,20 +6135,20 @@ describe('DependencyGraph', function() {
});
});
pit('calls back for each finished module', () => {
it('calls back for each finished module', () => {
return getDependencies().then(() =>
expect(onProgress.mock.calls.length).toBe(8)
);
});
pit('increases the number of finished modules in steps of one', () => {
it('increases the number of finished modules in steps of one', () => {
return getDependencies().then(() => {
const increments = onProgress.mock.calls.map(([finished]) => finished);
expect(increments).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
});
});
pit('adds the number of discovered modules to the number of total modules', () => {
it('adds the number of discovered modules to the number of total modules', () => {
return getDependencies().then(() => {
const increments = onProgress.mock.calls.map(([, total]) => total);
expect(increments).toEqual([3, 5, 6, 6, 7, 7, 8, 8]);
@ -6062,7 +6162,7 @@ describe('DependencyGraph', function() {
DependencyGraph = require('../index');
});
pit('allows setting dependencies for asset modules', () => {
it('allows setting dependencies for asset modules', () => {
const assetDependencies = ['arbitrary', 'dependencies'];
setMockFileSystem({
@ -6121,7 +6221,7 @@ describe('DependencyGraph', function() {
roots: ['/root'],
});
moduleReadDeferreds = {};
callDeferreds = [defer()/* a.js */, defer()/* b.js */];
callDeferreds = [defer(), defer()]; // [a.js, b.js]
Module.prototype.read = jest.genMockFn().mockImplementation(function() {
const returnValue = moduleRead.apply(this, arguments);
@ -6143,7 +6243,7 @@ describe('DependencyGraph', function() {
Module.prototype.read = moduleRead;
});
pit('produces a deterministic tree if the "a" module resolves first', () => {
it('produces a deterministic tree if the "a" module resolves first', () => {
const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js');
return Promise.all(callDeferreds.map(deferred => deferred.promise))
@ -6168,7 +6268,7 @@ describe('DependencyGraph', function() {
});
});
pit('produces a deterministic tree if the "b" module resolves first', () => {
it('produces a deterministic tree if the "b" module resolves first', () => {
const dependenciesPromise = getOrderedDependenciesAsJSON(dependencyGraph, 'index.js');
return Promise.all(callDeferreds.map(deferred => deferred.promise))

7
packager/react-packager/src/node-haste/__tests__/Module-test.js vendored
View File

@ -78,17 +78,16 @@ describe('Module', () => {
const createJSONModule =
(options) => createModule({...options, file: '/root/package.json'});
beforeEach(function(done) {
beforeEach(function() {
process.platform = 'linux';
cache = createCache();
fastfs = new Fastfs(
'test',
['/root'],
fileWatcher,
{crawling: Promise.resolve([fileName, '/root/package.json']), ignore: []},
['/root/index.js', '/root/package.json'],
{ignore: []},
);
fastfs.build().then(done);
});
describe('Module ID', () => {

12
packager/react-packager/src/node-haste/__tests__/fastfs-integrated-test.js vendored
View File

@ -22,13 +22,17 @@ const contents = fs.readFileSync(fileName, 'utf-8');
describe('fastfs:', function() {
let fastfs;
const crawling = Promise.resolve([fileName]);
const roots = [__dirname];
const watcher = new EventEmitter();
beforeEach(function(done) {
fastfs = new Fastfs('arbitrary', roots, watcher, {crawling});
fastfs.build().then(done);
beforeEach(function() {
fastfs = new Fastfs(
'arbitrary',
roots,
watcher,
[`${__dirname}/fastfs-data`],
{}
);
});
describe('partial reading', () => {

13
packager/react-packager/src/node-haste/crawlers/index.js vendored
View File

@ -1,13 +0,0 @@
'use strict';
const nodeCrawl = require('./node');
const watchmanCrawl = require('./watchman');
function crawl(roots, options) {
const {fileWatcher} = options;
return (fileWatcher ? fileWatcher.isWatchman() : Promise.resolve(false)).then(
isWatchman => isWatchman ? watchmanCrawl(roots, options) : nodeCrawl(roots, options)
);
}
module.exports = crawl;

61
packager/react-packager/src/node-haste/crawlers/node.js vendored
View File

@ -1,61 +0,0 @@
'use strict';
const denodeify = require('denodeify');
const debug = require('debug')('ReactNativePackager:DependencyGraph');
const fs = require('graceful-fs');
const path = require('../fastpath');
const readDir = denodeify(fs.readdir);
const stat = denodeify(fs.stat);
function nodeRecReadDir(roots, {ignore, exts}) {
const queue = roots.slice();
const retFiles = [];
const extPattern = new RegExp(
'\.(' + exts.join('|') + ')$'
);
function search() {
const currDir = queue.shift();
if (!currDir) {
return Promise.resolve();
}
return readDir(currDir)
.then(files => files.map(f => path.join(currDir, f)))
.then(files => Promise.all(
files.map(f => stat(f).catch(handleBrokenLink))
).then(stats => [
// Remove broken links.
files.filter((file, i) => !!stats[i]),
stats.filter(Boolean),
]))
.then(([files, stats]) => {
files.forEach((filePath, i) => {
if (ignore(filePath)) {
return;
}
if (stats[i].isDirectory()) {
queue.push(filePath);
return;
}
if (filePath.match(extPattern)) {
retFiles.push(path.resolve(filePath));
}
});
return search();
});
}
return search().then(() => retFiles);
}
function handleBrokenLink(e) {
debug('WARNING: error stating, possibly broken symlink', e.message);
return Promise.resolve();
}
module.exports = nodeRecReadDir;

76
packager/react-packager/src/node-haste/crawlers/watchman.js vendored
View File

@ -1,76 +0,0 @@
'use strict';
const denodeify = require('denodeify');
const path = require('../fastpath');
const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html';
function watchmanRecReadDir(roots, {ignore, fileWatcher, exts}) {
const files = [];
return Promise.all(
roots.map(
root => fileWatcher.getWatcherForRoot(root)
)
).then(
watchers => {
// All watchman roots for all watches we have.
const watchmanRoots = watchers.map(
watcher => watcher.watchProjectInfo.root
);
// Actual unique watchers (because we use watch-project we may end up with
// duplicate "real" watches, and that's by design).
// TODO(amasad): push this functionality into the `FileWatcher`.
const uniqueWatchers = watchers.filter(
(watcher, i) => watchmanRoots.indexOf(watcher.watchProjectInfo.root) === i
);
return Promise.all(
uniqueWatchers.map(watcher => {
const watchedRoot = watcher.watchProjectInfo.root;
// Build up an expression to filter the output by the relevant roots.
const dirExpr = ['anyof'];
for (let i = 0; i < roots.length; i++) {
const root = roots[i];
if (isDescendant(watchedRoot, root)) {
dirExpr.push(['dirname', path.relative(watchedRoot, root)]);
}
}
const cmd = denodeify(watcher.client.command.bind(watcher.client));
return cmd(['query', watchedRoot, {
suffix: exts,
expression: ['allof', ['type', 'f'], 'exists', dirExpr],
fields: ['name'],
}]).then(resp => {
if ('warning' in resp) {
console.warn('watchman warning: ', resp.warning);
}
resp.files.forEach(filePath => {
filePath = watchedRoot + path.sep + filePath;
if (!ignore(filePath)) {
files.push(filePath);
}
return false;
});
});
})
);
}).then(
() => files,
error => {
throw new Error(
`Watchman error: ${error.message.trim()}. Make sure watchman ` +
`is running for this project. See ${watchmanURL}.`
);
}
);
}
function isDescendant(root, child) {
return root === child || child.startsWith(root + path.sep);
}
module.exports = watchmanRecReadDir;

8
packager/react-packager/src/node-haste/fastfs.js vendored
View File

@ -20,7 +20,7 @@ const stat = denodeify(fs.stat);
const NOT_FOUND_IN_ROOTS = 'NotFoundInRootsError';
class Fastfs extends EventEmitter {
constructor(name, roots, fileWatcher, {ignore, crawling, activity}) {
constructor(name, roots, fileWatcher, files, {ignore, activity}) {
super();
this._name = name;
this._fileWatcher = fileWatcher;
@ -37,14 +37,9 @@ class Fastfs extends EventEmitter {
return new File(root, true);
});
this._fastPaths = Object.create(null);
this._crawling = crawling;
this._activity = activity;
}
build() {
return this._crawling.then(files => {
let fastfsActivity;
const activity = this._activity;
if (activity) {
fastfsActivity = activity.startEvent(
'Building in-memory fs for ' + this._name,
@ -75,7 +70,6 @@ class Fastfs extends EventEmitter {
if (this._fileWatcher) {
this._fileWatcher.on('all', this._processFileChange.bind(this));
}
});
}
stat(filePath) {

108
packager/react-packager/src/node-haste/index.js vendored
View File

@ -11,10 +11,10 @@
const Cache = require('./Cache');
const Fastfs = require('./fastfs');
const FileWatcher = require('./FileWatcher');
const JestHasteMap = require('jest-haste-map');
const Module = require('./Module');
const ModuleCache = require('./ModuleCache');
const Polyfill = require('./Polyfill');
const crawl = require('./crawlers');
const extractRequires = require('./lib/extractRequires');
const getAssetDataFromName = require('./lib/getAssetDataFromName');
const getInverseDependencies = require('./lib/getInverseDependencies');
@ -23,6 +23,7 @@ const isAbsolutePath = require('absolute-path');
const replacePatterns = require('./lib/replacePatterns');
const path = require('./fastpath');
const util = require('util');
const os = require('os');
const DependencyGraphHelpers = require('./DependencyGraph/DependencyGraphHelpers');
const ResolutionRequest = require('./DependencyGraph/ResolutionRequest');
const ResolutionResponse = require('./DependencyGraph/ResolutionResponse');
@ -57,6 +58,10 @@ class DependencyGraph {
assetDependencies,
moduleOptions,
extraNodeModules,
// additional arguments for jest-haste-map
useWatchman,
maxWorkers,
resetCache,
}) {
this._opts = {
activity: activity || defaultActivity,
@ -78,6 +83,10 @@ class DependencyGraph {
cacheTransformResults: true,
},
extraNodeModules,
// additional arguments for jest-haste-map & defaults
useWatchman: useWatchman !== false,
maxWorkers,
resetCache,
};
this._cache = cache;
this._assetDependencies = assetDependencies;
@ -90,6 +99,22 @@ class DependencyGraph {
return this._loading;
}
const mw = this._opts.maxWorkers;
const haste = new JestHasteMap({
extensions: this._opts.extensions.concat(this._opts.assetExts),
ignorePattern: {test: this._opts.ignoreFilePath},
maxWorkers: typeof mw === 'number' && mw >= 1 ? mw : getMaxWorkers(),
mocksPattern: '',
name: 'react-native-packager',
platforms: Array.from(this._opts.platforms),
providesModuleNodeModules: this._opts.providesModuleNodeModules,
resetCache: this._opts.resetCache,
retainAllFiles: true,
roots: this._opts.roots.concat(this._opts.assetRoots_DEPRECATED),
useWatchman: this._opts.useWatchman,
});
this._loading = haste.build().then(hasteMap => {
const {activity} = this._opts;
const depGraphActivity = activity.startEvent(
'Initializing Packager',
@ -98,28 +123,16 @@ class DependencyGraph {
telemetric: true,
},
);
const crawlActivity = activity.startEvent(
'Crawling File System',
null,
{
telemetric: true,
},
);
const allRoots = this._opts.roots.concat(this._opts.assetRoots_DEPRECATED);
this._crawling = crawl(allRoots, {
ignore: this._opts.ignoreFilePath,
exts: this._opts.extensions.concat(this._opts.assetExts),
fileWatcher: this._opts.fileWatcher,
});
this._crawling.then((files) => activity.endEvent(crawlActivity));
const hasteFSFiles = hasteMap.hasteFS.getAllFiles();
this._fastfs = new Fastfs(
'JavaScript',
this._opts.roots,
this._opts.fileWatcher,
hasteFSFiles,
{
ignore: this._opts.ignoreFilePath,
crawling: this._crawling,
activity: activity,
}
);
@ -145,21 +158,28 @@ class DependencyGraph {
platforms: this._opts.platforms,
});
const escapePath = (p: string) => {
return (path.sep === '\\') ? p.replace(/(\/|\\(?!\.))/g, '\\\\') : p;
};
const assetPattern =
new RegExp('^' + this._opts.assetRoots_DEPRECATED.map(escapePath).join('|'));
const assetFiles = hasteMap.hasteFS.matchFiles(assetPattern);
this._deprecatedAssetMap = new DeprecatedAssetMap({
fsCrawl: this._crawling,
roots: this._opts.assetRoots_DEPRECATED,
helpers: this._helpers,
fileWatcher: this._opts.fileWatcher,
ignoreFilePath: this._opts.ignoreFilePath,
assetExts: this._opts.assetExts,
activity: this._opts.activity,
enabled: this._opts.enableAssetMap,
platforms: this._opts.platforms,
files: assetFiles,
});
this._fastfs.on('change', (type, filePath, root, fstat) => {
if (assetPattern.test(path.join(root, filePath))) {
this._deprecatedAssetMap.processFileChange(type, filePath, root, fstat);
}
});
this._loading = Promise.all([
this._fastfs.build()
.then(() => {
const hasteActivity = activity.startEvent(
'Building Haste Map',
null,
@ -167,16 +187,11 @@ class DependencyGraph {
telemetric: true,
},
);
return this._hasteMap.build().then(map => {
return this._hasteMap.build().then(
map => {
activity.endEvent(hasteActivity);
return map;
});
}),
this._deprecatedAssetMap.build(),
]).then(
response => {
activity.endEvent(depGraphActivity);
return response[0]; // Return the haste map
return map;
},
err => {
const error = new Error(
@ -187,6 +202,7 @@ class DependencyGraph {
throw error;
}
);
});
return this._loading;
}
@ -312,7 +328,7 @@ class DependencyGraph {
this._loading = this._hasteMap.build();
} else {
this._loading = this._hasteMap.processFileChange(type, absPath);
this._loading.catch((e) => this._hasteMapError = e);
this._loading.catch((e) => {this._hasteMapError = e;});
}
return this._loading;
};
@ -351,4 +367,28 @@ function NotFoundError() {
}
util.inherits(NotFoundError, Error);
function getMaxWorkers() {
const cores = os.cpus().length;
if (cores <= 1) {
// oh well...
return 1;
}
if (cores <= 4) {
// don't starve the CPU while still reading reasonably rapidly
return cores - 1;
}
if (cores <= 8) {
// empirical testing showed massive diminishing returns when going over
// 4 or 5 workers on 8-core machines
return Math.floor(cores * 0.75) - 1;
}
// pretty much guesswork
if (cores < 24) {
return Math.floor(3 / 8 * cores + 3);
}
return cores / 2;
}
module.exports = DependencyGraph;