[react-packager] Add support for nested node_modules

Summary:
@public
The packager's resolver started out imitating node-haste, which meant that we didn't support nested modules. Now this is a problem. Bigger projects are bound to have versions of different versions of the same package at different levels of the dependency tree. This
makes loading dependencies lazy for node_modules and implements the node resolution algorithm. However, it also mantains that some
modules are still "haste" format, which currently defaults to "react-native" and "react-tools".

Finally, this means ~5 seconds speed up on every server start. This should also have a big impact on open source users with projects with big node_modules.

Test Plan:
1- test the app with --reset-cache
2- click around test and production apps
3- update the OSS library
4- create a new project
5- npm install multiple modules
6- create some version conflict in your project
7- make sure we do the "right" thing
8- test file changes to make sure it works
This commit is contained in:
Amjad Masad 2015-06-03 11:39:39 -07:00
parent 435125f4a0
commit 5a191dadfc
6 changed files with 2125 additions and 377 deletions

View File

@ -8,6 +8,9 @@
*/
'use strict';
var Promise = require('bluebird');
var isAbsolutePath = require('absolute-path');
function ModuleDescriptor(fields) {
if (!fields.id) {
throw new Error('Missing required fields id');
@ -17,17 +20,13 @@ function ModuleDescriptor(fields) {
if (!fields.path) {
throw new Error('Missing required fields path');
}
if (!isAbsolutePath(fields.path)) {
throw new Error('Expected absolute path but found: ' + fields.path);
}
this.path = fields.path;
if (!fields.dependencies) {
throw new Error('Missing required fields dependencies');
}
this.dependencies = fields.dependencies;
this.resolveDependency = fields.resolveDependency;
this.entry = fields.entry || false;
this.isPolyfill = fields.isPolyfill || false;
this.isAsset_DEPRECATED = fields.isAsset_DEPRECATED || false;
@ -50,12 +49,30 @@ function ModuleDescriptor(fields) {
this._fields = fields;
}
ModuleDescriptor.prototype.loadDependencies = function(loader) {
if (!this.dependencies) {
if (this._loadingDependencies) {
return this._loadingDependencies;
}
var self = this;
this._loadingDependencies = loader(this).then(function(dependencies) {
self.dependencies = dependencies;
});
return this._loadingDependencies;
}
return Promise.resolve(this.dependencies);
};
ModuleDescriptor.prototype.toJSON = function() {
return {
id: this.id,
path: this.path,
dependencies: this.dependencies
};
var ret = {};
Object.keys(this).forEach(function(prop) {
if (prop[0] !== '_' && typeof this[prop] !== 'function') {
ret[prop] = this[prop];
}
}, this);
return ret;
};
module.exports = ModuleDescriptor;

View File

@ -0,0 +1,109 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest
.dontMock('absolute-path')
.dontMock('../ModuleDescriptor');
describe('ModuleDescriptor', function() {
var ModuleDescriptor;
var Promise;
beforeEach(function() {
ModuleDescriptor = require('../ModuleDescriptor');
Promise = require('bluebird');
});
describe('constructor', function() {
it('should validate fields', function() {
/* eslint no-new:0*/
expect(function() {
new ModuleDescriptor({});
}).toThrow();
expect(function() {
new ModuleDescriptor({
id: 'foo',
});
}).toThrow();
expect(function() {
new ModuleDescriptor({
id: 'foo',
path: 'foo',
});
}).toThrow();
expect(function() {
new ModuleDescriptor({
id: 'foo',
path: '/foo',
isAsset: true,
});
}).toThrow();
var m = new ModuleDescriptor({
id: 'foo',
path: '/foo',
isAsset: true,
resolution: 1,
});
expect(m.toJSON()).toEqual({
altId:undefined,
dependencies: undefined,
isAsset_DEPRECATED: false,
isJSON: undefined,
isPolyfill: false,
id: 'foo',
path: '/foo',
isAsset: true,
resolution: 1,
});
});
});
describe('loadDependencies', function() {
pit('should load dependencies', function() {
var mod = new ModuleDescriptor({
id: 'foo',
path: '/foo',
isAsset: true,
resolution: 1,
});
return mod.loadDependencies(function() {
return Promise.resolve([1, 2]);
}).then(function() {
expect(mod.dependencies).toEqual([1, 2]);
});
});
pit('should load cached dependencies', function() {
var mod = new ModuleDescriptor({
id: 'foo',
path: '/foo',
isAsset: true,
resolution: 1,
});
return mod.loadDependencies(function() {
return Promise.resolve([1, 2]);
}).then(function() {
return mod.loadDependencies(function() {
throw new Error('no!');
});
}).then(function() {
expect(mod.dependencies).toEqual([1, 2]);
});
});
});
});

View File

@ -1,3 +1,6 @@
// TODO
// Fix it to work with tests
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
@ -19,6 +22,7 @@ var debug = require('debug')('DependecyGraph');
var util = require('util');
var declareOpts = require('../../../lib/declareOpts');
var getAssetDataFromName = require('../../../lib/getAssetDataFromName');
var crypto = require('crypto');
var readFile = Promise.promisify(fs.readFile);
var readDir = Promise.promisify(fs.readdir);
@ -45,7 +49,11 @@ var validateOpts = declareOpts({
assetExts: {
type: 'array',
required: true,
}
},
_providesModuleNodeModules: {
type: 'array',
default: ['react-tools', 'react-native'],
},
});
function DependecyGraph(options) {
@ -69,6 +77,8 @@ function DependecyGraph(options) {
'\.(' + ['js', 'json'].concat(this._assetExts).join('|') + ')$'
);
this._providesModuleNodeModules = opts._providesModuleNodeModules;
// Kick off the search process to precompute the dependency graph.
this._init();
}
@ -90,59 +100,101 @@ DependecyGraph.prototype.load = function() {
* Given an entry file return an array of all the dependent module descriptors.
*/
DependecyGraph.prototype.getOrderedDependencies = function(entryPath) {
var absolutePath = this._getAbsolutePath(entryPath);
if (absolutePath == null) {
throw new NotFoundError(
'Cannot find entry file %s in any of the roots: %j',
entryPath,
this._roots
);
}
return this.load().then(function() {
var absolutePath = this._getAbsolutePath(entryPath);
if (absolutePath == null) {
throw new NotFoundError(
'Cannot find entry file %s in any of the roots: %j',
entryPath,
this._roots
);
}
var module = this._graph[absolutePath];
if (module == null) {
throw new Error('Module with path "' + entryPath + '" is not in graph');
}
var module = this._graph[absolutePath];
if (module == null) {
throw new Error('Module with path "' + entryPath + '" is not in graph');
}
var self = this;
var deps = [];
var visited = Object.create(null);
var self = this;
var deps = [];
var visited = Object.create(null);
// Node haste sucks. Id's aren't unique. So to make sure our entry point
// is the thing that ends up in our dependency list.
var graphMap = Object.create(this._moduleById);
graphMap[module.id] = module;
// Node haste sucks. Id's aren't unique. So to make sure our entry point
// is the thing that ends up in our dependency list.
var graphMap = Object.create(this._moduleById);
graphMap[module.id] = module;
// Recursively collect the dependency list.
function collect(module) {
deps.push(module);
// Recursively collect the dependency list.
function collect(mod) {
deps.push(mod);
module.dependencies.forEach(function(name) {
var id = sansExtJs(name);
var dep = self.resolveDependency(module, id);
if (dep == null) {
debug(
'WARNING: Cannot find required module `%s` from module `%s`.',
name,
module.id
);
return;
if (mod.dependencies == null) {
return mod.loadDependencies(function() {
return readFile(mod.path, 'utf8').then(function(content) {
return extractRequires(content);
});
}).then(function() {
return iter(mod);
});
}
if (!visited[dep.id]) {
visited[dep.id] = true;
collect(dep);
}
return iter(mod);
}
function iter(mod) {
var p = Promise.resolve();
mod.dependencies.forEach(function(name) {
var id = sansExtJs(name);
var dep = self.resolveDependency(mod, id);
if (dep == null) {
debug(
'WARNING: Cannot find required module `%s` from module `%s`.',
name,
mod.id
);
return;
}
p = p.then(function() {
if (!visited[realId(dep)]) {
visited[realId(dep)] = true;
return collect(dep);
}
return null;
});
});
return p;
}
visited[realId(module)] = true;
return collect(module).then(function() {
return deps;
});
}
visited[module.id] = true;
collect(module);
return deps;
}.bind(this));
};
function browserFieldRedirect(packageJson, modulePath, isMain) {
if (packageJson.browser && typeof packageJson.browser === 'object') {
if (isMain) {
var tmpMain = packageJson.browser[modulePath] ||
packageJson.browser[sansExtJs(modulePath)] ||
packageJson.browser[withExtJs(modulePath)];
if (tmpMain) {
return tmpMain;
}
} else {
var relPath = './' + path.relative(packageJson._root, modulePath);
var tmpModulePath = packageJson.browser[withExtJs(relPath)] ||
packageJson.browser[sansExtJs(relPath)];
if (tmpModulePath) {
return path.join(packageJson._root, tmpModulePath);
}
}
}
return modulePath;
}
/**
* Given a module descriptor `fromModule` return the module descriptor for
* the required module `depModuleId`. It could be top-level or relative,
@ -157,7 +209,7 @@ DependecyGraph.prototype.resolveDependency = function(
// Process DEPRECATED global asset requires.
if (assetMatch && assetMatch[1]) {
if (!this._assetMap_DEPRECATED[assetMatch[1]]) {
debug('WARINING: Cannot find asset:', assetMatch[1]);
debug('WARNING: Cannot find asset:', assetMatch[1]);
return null;
}
return this._assetMap_DEPRECATED[assetMatch[1]];
@ -177,19 +229,39 @@ DependecyGraph.prototype.resolveDependency = function(
depModuleId = fromPackageJson.browser[depModuleId];
}
var packageName = depModuleId.replace(/\/.+/, '');
packageJson = this._lookupNodePackage(fromModule.path, packageName);
if (packageJson != null && packageName !== depModuleId) {
modulePath = path.join(
packageJson._root,
path.relative(packageName, depModuleId)
);
modulePath = browserFieldRedirect(packageJson, modulePath);
dep = this._graph[withExtJs(modulePath)];
if (dep != null) {
return dep;
}
}
// `depModuleId` is simply a top-level `providesModule`.
// `depModuleId` is a package module but given the full path from the
// package, i.e. package_name/module_name
if (this._moduleById[sansExtJs(depModuleId)]) {
if (packageJson == null && this._moduleById[sansExtJs(depModuleId)]) {
return this._moduleById[sansExtJs(depModuleId)];
}
// `depModuleId` is a package and it's depending on the "main" resolution.
packageJson = this._packagesById[depModuleId];
// We are being forgiving here and raising an error because we could be
// processing a file that uses it's own require system.
if (packageJson == null) {
// `depModuleId` is a package and it's depending on the "main" resolution.
packageJson = this._packagesById[depModuleId];
}
// We are being forgiving here and not raising an error because we could be
// processing a file that uses it's own require system.
if (packageJson == null || packageName !== depModuleId) {
debug(
'WARNING: Cannot find required module `%s` from module `%s`.',
depModuleId,
@ -198,33 +270,8 @@ DependecyGraph.prototype.resolveDependency = function(
return null;
}
var main;
// We prioritize the `browser` field if it's a module path.
if (typeof packageJson.browser === 'string') {
main = packageJson.browser;
} else {
main = packageJson.main || 'index';
}
// If there is a mapping for main in the `browser` field.
if (packageJson.browser && typeof packageJson.browser === 'object') {
var tmpMain = packageJson.browser[main] ||
packageJson.browser[withExtJs(main)] ||
packageJson.browser[sansExtJs(main)];
if (tmpMain) {
main = tmpMain;
}
}
modulePath = withExtJs(path.join(packageJson._root, main));
dep = this._graph[modulePath];
// Some packages use just a dir and rely on an index.js inside that dir.
if (dep == null) {
dep = this._graph[path.join(packageJson._root, main, 'index.js')];
}
// We are requiring node or a haste package via it's main file.
dep = this._resolvePackageMain(packageJson);
if (dep == null) {
throw new Error(
'Cannot find package main file for package: ' + packageJson._root
@ -248,15 +295,7 @@ DependecyGraph.prototype.resolveDependency = function(
// modulePath: /x/y/a/b
var dir = path.dirname(fromModule.path);
modulePath = path.join(dir, depModuleId);
if (packageJson.browser && typeof packageJson.browser === 'object') {
var relPath = './' + path.relative(packageJson._root, modulePath);
var tmpModulePath = packageJson.browser[withExtJs(relPath)] ||
packageJson.browser[sansExtJs(relPath)];
if (tmpModulePath) {
modulePath = path.join(packageJson._root, tmpModulePath);
}
}
modulePath = browserFieldRedirect(packageJson, modulePath);
// JS modules can be required without extensios.
if (!this._isFileAsset(modulePath) && !modulePath.match(/\.json$/)) {
@ -291,6 +330,25 @@ DependecyGraph.prototype.resolveDependency = function(
}
};
DependecyGraph.prototype._resolvePackageMain = function(packageJson) {
var main;
// We prioritize the `browser` field if it's a module path.
if (typeof packageJson.browser === 'string') {
main = packageJson.browser;
} else {
main = packageJson.main || 'index';
}
// If there is a mapping for main in the `browser` field.
main = browserFieldRedirect(packageJson, main, true);
var modulePath = withExtJs(path.join(packageJson._root, main));
return this._graph[modulePath] ||
// Some packages use just a dir and rely on an index.js inside that dir.
this._graph[path.join(packageJson._root, main, 'index.js')];
};
/**
* Intiates the filewatcher and kicks off the search process.
*/
@ -339,9 +397,9 @@ DependecyGraph.prototype._search = function() {
});
var processing = self._findAndProcessPackage(files, dir)
.then(function() {
return Promise.all(modulePaths.map(self._processModule.bind(self)));
});
.then(function() {
return Promise.all(modulePaths.map(self._processModule.bind(self)));
});
return Promise.all([
processing,
@ -358,10 +416,8 @@ DependecyGraph.prototype._search = function() {
* and update indices.
*/
DependecyGraph.prototype._findAndProcessPackage = function(files, root) {
var self = this;
var packagePath;
for (var i = 0; i < files.length ; i++) {
for (var i = 0; i < files.length; i++) {
var file = files[i];
if (path.basename(file) === 'package.json') {
packagePath = file;
@ -406,12 +462,16 @@ DependecyGraph.prototype._processPackage = function(packagePath) {
DependecyGraph.prototype._addPackageToIndices = function(packageJson) {
this._packageByRoot[packageJson._root] = packageJson;
this._packagesById[packageJson.name] = packageJson;
if (!this._isInNodeModules(packageJson._root)) {
this._packagesById[packageJson.name] = packageJson;
}
};
DependecyGraph.prototype._removePackageFromIndices = function(packageJson) {
delete this._packageByRoot[packageJson._root];
delete this._packagesById[packageJson.name];
if (!this._isInNodeModules(packageJson._root)) {
delete this._packagesById[packageJson.name];
}
};
/**
@ -441,6 +501,14 @@ DependecyGraph.prototype._processModule = function(modulePath) {
return Promise.resolve(module);
}
if (this._isInNodeModules(modulePath)) {
moduleData.id = this._lookupName(modulePath);
moduleData.dependencies = null;
module = new ModuleDescriptor(moduleData);
this._updateGraphWithModule(module);
return Promise.resolve(module);
}
var self = this;
return readFile(modulePath, 'utf8')
.then(function(content) {
@ -484,6 +552,10 @@ DependecyGraph.prototype._deleteModule = function(module) {
// Others may keep a reference so we mark it as deleted.
module.deleted = true;
if (this._isInNodeModules(module.path)) {
return;
}
// Haste allows different module to have the same id.
if (this._moduleById[module.id] === module) {
delete this._moduleById[module.id];
@ -504,6 +576,10 @@ DependecyGraph.prototype._updateGraphWithModule = function(module) {
this._graph[module.path] = module;
if (this._isInNodeModules(module.path)) {
return;
}
if (this._moduleById[module.id]) {
debug(
'WARNING: Top-level module name conflict `%s`.\n' +
@ -527,28 +603,14 @@ DependecyGraph.prototype._updateGraphWithModule = function(module) {
* Find the nearest package to a module.
*/
DependecyGraph.prototype._lookupPackage = function(modulePath) {
var packageByRoot = this._packageByRoot;
return lookupPackage(path.dirname(modulePath), this._packageByRoot);
};
/**
* Auxiliary function to recursively lookup a package.
*/
function lookupPackage(currDir) {
// ideally we stop once we're outside root and this can be a simple child
// dir check. However, we have to support modules that was symlinked inside
// our project root.
if (currDir === '/') {
return null;
} else {
var packageJson = packageByRoot[currDir];
if (packageJson) {
return packageJson;
} else {
return lookupPackage(path.dirname(currDir));
}
}
}
return lookupPackage(path.dirname(modulePath));
/**
* Find the nearest node package to a module.
*/
DependecyGraph.prototype._lookupNodePackage = function(startPath, packageName) {
return lookupNodePackage(path.dirname(startPath), this._packageByRoot, packageName);
};
/**
@ -573,12 +635,14 @@ DependecyGraph.prototype._processFileChange = function(
}
var isPackage = path.basename(filePath) === 'package.json';
var packageJson;
if (isPackage) {
packageJson = this._packageByRoot[path.dirname(absPath)];
}
if (eventType === 'delete') {
if (isPackage) {
var packageJson = this._packageByRoot[path.dirname(absPath)];
if (packageJson) {
this._removePackageFromIndices(packageJson);
}
if (isPackage && packageJson) {
this._removePackageFromIndices(packageJson);
} else {
var module = this._graph[absPath];
if (module == null) {
@ -591,7 +655,14 @@ DependecyGraph.prototype._processFileChange = function(
var self = this;
this._loading = this._loading.then(function() {
if (isPackage) {
return self._processPackage(absPath);
self._removePackageFromIndices(packageJson);
return self._processPackage(absPath)
.then(function(p) {
return self._resolvePackageMain(p);
})
.then(function(mainModule) {
return self._processModule(mainModule.path);
});
}
return self._processModule(absPath);
});
@ -675,6 +746,25 @@ DependecyGraph.prototype._isFileAsset = function(file) {
return this._assetExts.indexOf(extname(file)) !== -1;
};
DependecyGraph.prototype._isInNodeModules = function(file) {
var inNodeModules = file.indexOf('/node_modules/') !== -1;
if (!inNodeModules) {
return false;
}
var dirs = this._providesModuleNodeModules;
for (var i = 0; i < dirs.length; i++) {
var index = file.indexOf(dirs[i]);
if (index !== -1) {
return file.slice(index).indexOf('/node_modules/') !== -1;
}
}
return true;
};
/**
* Extract all required modules from a `code` string.
*/
@ -784,6 +874,54 @@ function extname(name) {
return path.extname(name).replace(/^\./, '');
}
function realId(module) {
if (module._realId) {
return module._realId;
}
var hash = crypto.createHash('md5');
hash.update(module.id);
hash.update(module.path);
Object.defineProperty(module, '_realId', { value: hash.digest('hex') });
return module._realId;
}
/**
* Auxiliary function to recursively lookup a package.
*/
function lookupPackage(currDir, packageByRoot) {
// ideally we stop once we're outside root and this can be a simple child
// dir check. However, we have to support modules that was symlinked inside
// our project root.
if (currDir === '/') {
return null;
} else {
var packageJson = packageByRoot[currDir];
if (packageJson) {
return packageJson;
} else {
return lookupPackage(path.dirname(currDir), packageByRoot);
}
}
}
/**
* Auxiliary function to recursively lookup a package.
*/
function lookupNodePackage(currDir, packageByRoot, packageName) {
if (currDir === '/') {
return null;
}
var packageRoot = path.join(currDir, 'node_modules', packageName);
var packageJson = packageByRoot[packageRoot];
if (packageJson) {
return packageJson;
} else {
return lookupNodePackage(path.dirname(currDir), packageByRoot, packageName);
}
}
function NotFoundError() {
Error.call(this);
Error.captureStackTrace(this, this.constructor);

View File

@ -40,7 +40,7 @@ describe('HasteDependencyResolver', function() {
// Is there a better way? How can I mock the prototype instead?
var depGraph = depResolver._depGraph;
depGraph.getOrderedDependencies.mockImpl(function() {
return deps;
return Promise.resolve(deps);
});
depGraph.load.mockImpl(function() {
return Promise.resolve();
@ -123,7 +123,7 @@ describe('HasteDependencyResolver', function() {
// Is there a better way? How can I mock the prototype instead?
var depGraph = depResolver._depGraph;
depGraph.getOrderedDependencies.mockImpl(function() {
return deps;
return Promise.resolve(deps);
});
depGraph.load.mockImpl(function() {
return Promise.resolve();
@ -207,7 +207,7 @@ describe('HasteDependencyResolver', function() {
// Is there a better way? How can I mock the prototype instead?
var depGraph = depResolver._depGraph;
depGraph.getOrderedDependencies.mockImpl(function() {
return deps;
return Promise.resolve(deps);
});
depGraph.load.mockImpl(function() {
return Promise.resolve();

View File

@ -91,9 +91,8 @@ HasteDependencyResolver.prototype.getDependencies = function(main, options) {
var depGraph = this._depGraph;
var self = this;
return depGraph.load()
.then(function() {
var dependencies = depGraph.getOrderedDependencies(main);
return depGraph.getOrderedDependencies(main)
.then(function(dependencies) {
var mainModuleId = dependencies[0].id;
self._prependPolyfillDependencies(dependencies, opts.dev);