Backed out changeset 35b573bece59

Reviewed By: davidaurelio

Differential Revision: D4461681

fbshipit-source-id: f5c45f713f2fe68443986a2822aa325b01dd5075
This commit is contained in:
Jean Lauliac 2017-01-26 03:41:09 -08:00 committed by Facebook Github Bot
parent 795c1d5148
commit 30f915e818
10 changed files with 221 additions and 168 deletions

View File

@ -16,12 +16,14 @@ const Logger = require('./src/Logger');
const debug = require('debug');
const invariant = require('fbjs/lib/invariant');
import type GlobalTransformCache from './src/lib/GlobalTransformCache';
import type {Reporter} from './src/lib/reporting';
exports.createServer = createServer;
exports.Logger = Logger;
type Options = {
globalTransformCache: ?GlobalTransformCache,
nonPersistent: boolean,
projectRoots: Array<string>,
reporter?: Reporter,
@ -29,6 +31,7 @@ type Options = {
};
type StrictOptions = {
globalTransformCache: ?GlobalTransformCache,
nonPersistent: boolean,
projectRoots: Array<string>,
reporter: Reporter,

View File

@ -39,6 +39,7 @@ import type Module from '../node-haste/Module';
import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse';
import type {Options as JSTransformerOptions, TransformOptions} from '../JSTransformer/worker/worker';
import type {Reporter} from '../lib/reporting';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
export type GetTransformOptions = (
mainModuleName: string,
@ -70,6 +71,7 @@ type Options = {
cacheVersion: string,
extraNodeModules: {},
getTransformOptions?: GetTransformOptions,
globalTransformCache: ?GlobalTransformCache,
moduleFormat: string,
platforms: Array<string>,
polyfillModuleNames: Array<string>,
@ -150,6 +152,7 @@ class Bundler {
blacklistRE: opts.blacklistRE,
cache: this._cache,
extraNodeModules: opts.extraNodeModules,
globalTransformCache: opts.globalTransformCache,
minifyCode: this._transformer.minify,
moduleFormat: opts.moduleFormat,
platforms: opts.platforms,

View File

@ -23,6 +23,7 @@ import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type {Reporter} from '../lib/reporting';
import type {TransformCode} from '../node-haste/Module';
import type Cache from '../node-haste/Cache';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
type MinifyCode = (filePath: string, code: string, map: SourceMap) =>
Promise<{code: string, map: SourceMap}>;
@ -32,6 +33,7 @@ type Options = {
blacklistRE?: RegExp,
cache: Cache,
extraNodeModules?: {},
globalTransformCache: ?GlobalTransformCache,
minifyCode: MinifyCode,
platforms: Array<string>,
polyfillModuleNames?: Array<string>,
@ -56,6 +58,7 @@ class Resolver {
assetExts: opts.assetExts,
cache: opts.cache,
extraNodeModules: opts.extraNodeModules,
globalTransformCache: opts.globalTransformCache,
ignoreFilePath: function(filepath) {
return filepath.indexOf('__tests__') !== -1 ||
(opts.blacklistRE != null && opts.blacklistRE.test(filepath));

View File

@ -33,6 +33,7 @@ import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionRes
import type Bundle from '../Bundler/Bundle';
import type {Reporter} from '../lib/reporting';
import type {GetTransformOptions} from '../Bundler';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
const {
createActionStartEntry,
@ -59,6 +60,7 @@ type Options = {
cacheVersion?: string,
extraNodeModules?: {},
getTransformOptions?: GetTransformOptions,
globalTransformCache: ?GlobalTransformCache,
moduleFormat?: string,
platforms?: Array<string>,
polyfillModuleNames?: Array<string>,
@ -207,6 +209,7 @@ class Server {
cacheVersion: options.cacheVersion || '1.0',
extraNodeModules: options.extraNodeModules || {},
getTransformOptions: options.getTransformOptions,
globalTransformCache: options.globalTransformCache,
moduleFormat: options.moduleFormat != null ? options.moduleFormat : 'haste',
platforms: options.platforms || defaults.platforms,
polyfillModuleNames: options.polyfillModuleNames || [],
@ -236,6 +239,7 @@ class Server {
const bundlerOpts = Object.create(this._opts);
bundlerOpts.assetServer = this._assetServer;
bundlerOpts.allowBundleUpdates = this._opts.watch;
bundlerOpts.globalTransformCache = options.globalTransformCache;
bundlerOpts.watch = this._opts.watch;
bundlerOpts.reporter = options.reporter;
this._bundler = new Bundler(bundlerOpts);

104
react-packager/src/lib/BatchProcessor.js vendored Normal file
View File

@ -0,0 +1,104 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const invariant = require('fbjs/lib/invariant');
type ProcessBatch<TItem, TResult> = (
batch: Array<TItem>,
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
) => mixed;
type BatchProcessorOptions = {
maximumDelayMs: number,
maximumItems: number,
concurrency: number,
};
/**
* We batch items together trying to minimize their processing, for example as
* network queries. For that we wait a small moment before processing a batch.
* We limit also the number of items we try to process in a single batch so that
* if we have many items pending in a short amount of time, we can start
* processing right away.
*/
class BatchProcessor<TItem, TResult> {
_options: BatchProcessorOptions;
_processBatch: ProcessBatch<TItem, TResult>;
_queue: Array<{
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
}>;
_timeoutHandle: ?number;
_currentProcessCount: number;
constructor(
options: BatchProcessorOptions,
processBatch: ProcessBatch<TItem, TResult>,
) {
this._options = options;
this._processBatch = processBatch;
this._queue = [];
this._timeoutHandle = null;
this._currentProcessCount = 0;
(this: any)._processQueue = this._processQueue.bind(this);
}
_processQueue() {
this._timeoutHandle = null;
while (
this._queue.length > 0 &&
this._currentProcessCount < this._options.concurrency
) {
this._currentProcessCount++;
const jobs = this._queue.splice(0, this._options.maximumItems);
const items = jobs.map(job => job.item);
this._processBatch(items, (error, results) => {
invariant(
results == null || results.length === items.length,
'Not enough results returned.',
);
for (let i = 0; i < items.length; ++i) {
jobs[i].callback(error, results && results[i]);
}
this._currentProcessCount--;
this._processQueueOnceReady();
});
}
}
_processQueueOnceReady() {
if (this._queue.length >= this._options.maximumItems) {
clearTimeout(this._timeoutHandle);
process.nextTick(this._processQueue);
return;
}
if (this._timeoutHandle == null) {
this._timeoutHandle = setTimeout(
this._processQueue,
this._options.maximumDelayMs,
);
}
}
queue(
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
) {
this._queue.push({item, callback});
this._processQueueOnceReady();
}
}
module.exports = BatchProcessor;

View File

@ -11,15 +11,17 @@
'use strict';
const BatchProcessor = require('./BatchProcessor');
const crypto = require('crypto');
const imurmurhash = require('imurmurhash');
const invariant = require('fbjs/lib/invariant');
const jsonStableStringify = require('json-stable-stringify');
const path = require('path');
const request = require('request');
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type {CachedResult} from './TransformCache';
import type {Reporter} from './reporting';
type FetchResultURIs = (
keys: Array<string>,
@ -38,95 +40,9 @@ type FetchProps = {
transformOptions: TransformOptions,
};
type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
type FetchCallback = (error?: Error, result?: ?CachedResult) => mixed;
type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
type ProcessBatch<TItem, TResult> = (
batch: Array<TItem>,
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
) => mixed;
type BatchProcessorOptions = {
maximumDelayMs: number,
maximumItems: number,
concurrency: number,
};
/**
* We batch keys together trying to make a smaller amount of queries. For that
* we wait a small moment before starting to fetch. We limit also the number of
* keys we try to fetch at once, so if we already have that many keys pending,
* we can start fetching right away.
*/
class BatchProcessor<TItem, TResult> {
_options: BatchProcessorOptions;
_processBatch: ProcessBatch<TItem, TResult>;
_queue: Array<{
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
}>;
_timeoutHandle: ?number;
_currentProcessCount: number;
constructor(
options: BatchProcessorOptions,
processBatch: ProcessBatch<TItem, TResult>,
) {
this._options = options;
this._processBatch = processBatch;
this._queue = [];
this._timeoutHandle = null;
this._currentProcessCount = 0;
(this: any)._processQueue = this._processQueue.bind(this);
}
_processQueue() {
this._timeoutHandle = null;
while (
this._queue.length > 0 &&
this._currentProcessCount < this._options.concurrency
) {
this._currentProcessCount++;
const jobs = this._queue.splice(0, this._options.maximumItems);
const items = jobs.map(job => job.item);
this._processBatch(items, (error, results) => {
invariant(
results == null || results.length === items.length,
'Not enough results returned.',
);
for (let i = 0; i < items.length; ++i) {
jobs[i].callback(error, results && results[i]);
}
this._currentProcessCount--;
this._processQueueOnceReady();
});
}
}
_processQueueOnceReady() {
if (this._queue.length >= this._options.maximumItems) {
clearTimeout(this._timeoutHandle);
process.nextTick(this._processQueue);
return;
}
if (this._timeoutHandle == null) {
this._timeoutHandle = setTimeout(
this._processQueue,
this._options.maximumDelayMs,
);
}
}
queue(
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
) {
this._queue.push({item, callback});
this._processQueueOnceReady();
}
}
type URI = string;
/**
@ -135,16 +51,25 @@ type URI = string;
*/
class KeyURIFetcher {
_fetchResultURIs: FetchResultURIs;
_batchProcessor: BatchProcessor<string, ?URI>;
_fetchResultURIs: FetchResultURIs;
_processError: (error: Error) => mixed;
/**
* When a batch request fails for some reason, we process the error locally
* and we proceed as if there were no result for these keys instead. That way
* a build will not fail just because of the cache.
*/
_processKeys(
keys: Array<string>,
callback: (error?: Error, keyURIs: Array<?URI>) => mixed,
) {
this._fetchResultURIs(keys, (error, URIsByKey) => {
if (error != null) {
this._processError(error);
}
const URIs = keys.map(key => URIsByKey && URIsByKey.get(key));
callback(error, URIs);
callback(undefined, URIs);
});
}
@ -152,13 +77,14 @@ class KeyURIFetcher {
this._batchProcessor.queue(key, callback);
}
constructor(fetchResultURIs: FetchResultURIs) {
constructor(fetchResultURIs: FetchResultURIs, processError: (error: Error) => mixed) {
this._fetchResultURIs = fetchResultURIs;
this._batchProcessor = new BatchProcessor({
maximumDelayMs: 10,
maximumItems: 500,
concurrency: 25,
}, this._processKeys.bind(this));
this._processError = processError;
}
}
@ -257,24 +183,46 @@ class TransformProfileSet {
}
}
/**
* One can enable the global cache by calling configure() from a custom CLI
* script. Eventually we may make it more flexible.
*/
class GlobalTransformCache {
_fetcher: KeyURIFetcher;
_store: ?KeyResultStore;
_profileSet: TransformProfileSet;
static _global: ?GlobalTransformCache;
_reporter: Reporter;
_retries: number;
_store: ?KeyResultStore;
/**
* If too many errors already happened, we just drop the additional errors.
*/
_processError(error: Error) {
if (this._retries <= 0) {
return;
}
this._reporter.update({type: 'global_cache_error', error});
--this._retries;
if (this._retries <= 0) {
this._reporter.update({type: 'global_cache_disabled', reason: 'too_many_errors'});
}
}
/**
* For using the global cache one needs to have some kind of central key-value
* store that gets prefilled using keyOf() and the transformed results. The
* fetching function should provide a mapping of keys to URIs. The files
* referred by these URIs contains the transform results. Using URIs instead
* of returning the content directly allows for independent and parallel
* fetching of each result, that may be arbitrarily large JSON blobs.
*/
constructor(
fetchResultURIs: FetchResultURIs,
storeResults: ?StoreResults,
profiles: Iterable<TransformProfile>,
reporter: Reporter,
) {
this._fetcher = new KeyURIFetcher(fetchResultURIs, this._processError.bind(this));
this._profileSet = new TransformProfileSet(profiles);
this._fetcher = new KeyURIFetcher(fetchResultURIs);
this._reporter = reporter;
this._retries = 4;
if (storeResults != null) {
this._store = new KeyResultStore(storeResults);
}
@ -319,8 +267,22 @@ class GlobalTransformCache {
});
}
/**
* Wrap `_fetchFromURI` with error logging, and return an empty result instead
* of errors. This is because the global cache is not critical to the normal
* packager operation.
*/
_tryFetchingFromURI(uri: string, callback: FetchCallback) {
this._fetchFromURI(uri, (error, result) => {
if (error != null) {
this._processError(error);
}
callback(undefined, result);
});
}
fetch(props: FetchProps, callback: FetchCallback) {
if (!this._profileSet.has(props.transformOptions)) {
if (this._retries <= 0 || !this._profileSet.has(props.transformOptions)) {
process.nextTick(callback);
return;
}
@ -332,7 +294,7 @@ class GlobalTransformCache {
callback();
return;
}
this._fetchFromURI(uri, callback);
this._tryFetchingFromURI(uri, callback);
}
});
}
@ -343,32 +305,6 @@ class GlobalTransformCache {
}
}
/**
* For using the global cache one needs to have some kind of central key-value
* store that gets prefilled using keyOf() and the transformed results. The
* fetching function should provide a mapping of keys to URIs. The files
* referred by these URIs contains the transform results. Using URIs instead
* of returning the content directly allows for independent fetching of each
* result.
*/
static configure(
fetchResultURIs: FetchResultURIs,
storeResults: ?StoreResults,
profiles: Iterable<TransformProfile>,
) {
GlobalTransformCache._global = new GlobalTransformCache(
fetchResultURIs,
storeResults,
profiles,
);
}
static get() {
return GlobalTransformCache._global;
}
}
GlobalTransformCache._global = null;
module.exports = GlobalTransformCache;

View File

@ -152,7 +152,8 @@ class TerminalReporter {
terminal.log(`${DEP_GRAPH_MESSAGE}, done.`);
break;
case 'global_cache_error':
reporting.logWarning(terminal, 'The global cache failed: %s', event.error.stack);
const message = JSON.stringify(event.error.message);
reporting.logWarning(terminal, 'the global cache failed: %s', message);
break;
case 'global_cache_disabled':
this._logCacheDisabled(event.reason);

View File

@ -11,7 +11,6 @@
'use strict';
const GlobalTransformCache = require('../lib/GlobalTransformCache');
const TransformCache = require('../lib/TransformCache');
const crypto = require('crypto');
@ -24,6 +23,7 @@ const jsonStableStringify = require('json-stable-stringify');
const {join: joinPath, relative: relativePath, extname} = require('path');
import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {SourceMap} from '../lib/SourceMap';
import type {ReadTransformProps} from '../lib/TransformCache';
import type {Reporter} from '../lib/reporting';
@ -51,14 +51,15 @@ export type Options = {
};
export type ConstructorArgs = {
cache: Cache,
depGraphHelpers: DependencyGraphHelpers,
globalTransformCache: ?GlobalTransformCache,
file: string,
moduleCache: ModuleCache,
cache: Cache,
transformCode: ?TransformCode,
transformCacheKey: ?string,
depGraphHelpers: DependencyGraphHelpers,
options: Options,
reporter: Reporter,
transformCacheKey: ?string,
transformCode: ?TransformCode,
};
class Module {
@ -73,22 +74,22 @@ class Module {
_depGraphHelpers: DependencyGraphHelpers;
_options: Options;
_reporter: Reporter;
_globalCache: ?GlobalTransformCache;
_docBlock: Promise<{id?: string, moduleDocBlock: {[key: string]: mixed}}>;
_readSourceCodePromise: Promise<string>;
_readPromises: Map<string, Promise<ReadResult>>;
static _globalCacheRetries: number;
constructor({
file,
moduleCache,
cache,
transformCode,
transformCacheKey,
depGraphHelpers,
reporter,
file,
globalTransformCache,
moduleCache,
options,
reporter,
transformCacheKey,
transformCode,
}: ConstructorArgs) {
if (!isAbsolutePath(file)) {
throw new Error('Expected file to be absolute path but got ' + file);
@ -108,6 +109,7 @@ class Module {
this._depGraphHelpers = depGraphHelpers;
this._options = options || {};
this._reporter = reporter;
this._globalCache = globalTransformCache;
this._readPromises = new Map();
}
@ -263,28 +265,18 @@ class Module {
cacheProps: ReadTransformProps,
callback: (error: ?Error, result: ?TransformedCode) => void,
) {
const globalCache = GlobalTransformCache.get();
const noMoreRetries = Module._globalCacheRetries <= 0;
if (globalCache == null || noMoreRetries) {
const {_globalCache} = this;
if (_globalCache == null) {
this._transformCodeForCallback(cacheProps, callback);
return;
}
globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
if (globalCacheError != null && Module._globalCacheRetries > 0) {
this._reporter.update({
type: 'global_cache_error',
error: globalCacheError,
});
Module._globalCacheRetries--;
if (Module._globalCacheRetries <= 0) {
this._reporter.update({
type: 'global_cache_disabled',
reason: 'too_many_errors',
});
}
_globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
if (globalCacheError) {
callback(globalCacheError);
return;
}
if (globalCachedResult == null) {
this._transformAndStoreCodeGlobally(cacheProps, globalCache, callback);
this._transformAndStoreCodeGlobally(cacheProps, _globalCache, callback);
return;
}
callback(undefined, globalCachedResult);
@ -375,8 +367,6 @@ class Module {
}
}
Module._globalCacheRetries = 4;
// use weak map to speed up hash creation of known objects
const knownHashes = new WeakMap();
function stableObjectHash(object) {

View File

@ -16,13 +16,11 @@ const Module = require('./Module');
const Package = require('./Package');
const Polyfill = require('./Polyfill');
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {Reporter} from '../lib/reporting';
import type Cache from './Cache';
import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers';
import type {
TransformCode,
Options as ModuleOptions,
} from './Module';
import type {TransformCode, Options as ModuleOptions} from './Module';
type GetClosestPackageFn = (filePath: string) => ?string;
@ -32,6 +30,7 @@ class ModuleCache {
_cache: Cache;
_depGraphHelpers: DependencyGraphHelpers;
_getClosestPackage: GetClosestPackageFn;
_globalTransformCache: ?GlobalTransformCache;
_moduleCache: {[filePath: string]: Module};
_moduleOptions: ModuleOptions;
_packageCache: {[filePath: string]: Package};
@ -47,22 +46,25 @@ class ModuleCache {
depGraphHelpers,
extractRequires,
getClosestPackage,
globalTransformCache,
moduleOptions,
reporter,
transformCacheKey,
transformCode,
reporter,
}: {
assetDependencies: Array<string>,
cache: Cache,
depGraphHelpers: DependencyGraphHelpers,
getClosestPackage: GetClosestPackageFn,
globalTransformCache: ?GlobalTransformCache,
moduleOptions: ModuleOptions,
reporter: Reporter,
transformCacheKey: string,
transformCode: TransformCode,
reporter: Reporter,
}, platforms: Set<string>) {
this._assetDependencies = assetDependencies;
this._getClosestPackage = getClosestPackage;
this._globalTransformCache = globalTransformCache;
this._cache = cache;
this._depGraphHelpers = depGraphHelpers;
this._moduleCache = Object.create(null);
@ -78,14 +80,15 @@ class ModuleCache {
getModule(filePath: string) {
if (!this._moduleCache[filePath]) {
this._moduleCache[filePath] = new Module({
file: filePath,
moduleCache: this,
cache: this._cache,
transformCode: this._transformCode,
transformCacheKey: this._transformCacheKey,
depGraphHelpers: this._depGraphHelpers,
file: filePath,
globalTransformCache: this._globalTransformCache,
moduleCache: this,
options: this._moduleOptions,
reporter: this._reporter,
transformCacheKey: this._transformCacheKey,
transformCode: this._transformCode,
});
}
return this._moduleCache[filePath];

View File

@ -38,6 +38,7 @@ const {
} = require('../Logger');
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {Reporter} from '../lib/reporting';
import type {
Options as ModuleOptions,
@ -53,6 +54,7 @@ class DependencyGraph {
extensions: Array<string>,
extraNodeModules: ?Object,
forceNodeFilesystemAPI: boolean,
globalTransformCache: ?GlobalTransformCache,
ignoreFilePath: (filePath: string) => boolean,
maxWorkers: ?number,
mocksPattern: mixed,
@ -87,6 +89,7 @@ class DependencyGraph {
extensions,
extraNodeModules,
forceNodeFilesystemAPI,
globalTransformCache,
ignoreFilePath,
maxWorkers,
mocksPattern,
@ -109,6 +112,7 @@ class DependencyGraph {
extensions?: ?Array<string>,
extraNodeModules: ?Object,
forceNodeFilesystemAPI?: boolean,
globalTransformCache: ?GlobalTransformCache,
ignoreFilePath: (filePath: string) => boolean,
maxWorkers?: ?number,
mocksPattern?: mixed,
@ -130,6 +134,7 @@ class DependencyGraph {
extensions: extensions || ['js', 'json'],
extraNodeModules,
forceNodeFilesystemAPI: !!forceNodeFilesystemAPI,
globalTransformCache,
ignoreFilePath: ignoreFilePath || (() => {}),
maxWorkers,
mocksPattern,
@ -186,6 +191,7 @@ class DependencyGraph {
this._moduleCache = new ModuleCache({
cache: this._cache,
globalTransformCache: this._opts.globalTransformCache,
transformCode: this._opts.transformCode,
transformCacheKey: this._opts.transformCacheKey,
depGraphHelpers: this._helpers,