status-im
/
metro
mirror of https://github.com/status-im/metro.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Revert stacked revisions on packager that causes packager to instacrash 

Reviewed By: fkgozali, bestander

Differential Revision: D4456911

fbshipit-source-id: 35b573bece59c59c0700ef2d8ab26c0b7150d3cf
This commit is contained in:
Jing Chen 2017-01-24 11:32:37 -08:00 committed by Facebook Github Bot
parent f6e068bcf0
commit cbc808dc86
10 changed files with 168 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
react-packager/react-packager.js vendored
View File

@ -16,14 +16,12 @@ const Logger = require('./src/Logger');
const debug = require('debug');
const invariant = require('fbjs/lib/invariant');
import type GlobalTransformCache from './src/lib/GlobalTransformCache';
import type {Reporter} from './src/lib/reporting';
exports.createServer = createServer;
exports.Logger = Logger;
type Options = {
globalTransformCache: ?GlobalTransformCache,
nonPersistent: boolean,
projectRoots: Array<string>,
reporter?: Reporter,
@ -31,7 +29,6 @@ type Options = {
};
type StrictOptions = {
globalTransformCache: ?GlobalTransformCache,
nonPersistent: boolean,
projectRoots: Array<string>,
reporter: Reporter,

3
react-packager/src/Bundler/index.js vendored
View File

@ -39,7 +39,6 @@ import type Module from '../node-haste/Module';
import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse';
import type {Options as JSTransformerOptions, TransformOptions} from '../JSTransformer/worker/worker';
import type {Reporter} from '../lib/reporting';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
export type GetTransformOptions = (
mainModuleName: string,
@ -71,7 +70,6 @@ type Options = {
cacheVersion: string,
extraNodeModules: {},
getTransformOptions?: GetTransformOptions,
globalTransformCache: ?GlobalTransformCache,
moduleFormat: string,
platforms: Array<string>,
polyfillModuleNames: Array<string>,
@ -152,7 +150,6 @@ class Bundler {
blacklistRE: opts.blacklistRE,
cache: this._cache,
extraNodeModules: opts.extraNodeModules,
globalTransformCache: opts.globalTransformCache,
minifyCode: this._transformer.minify,
moduleFormat: opts.moduleFormat,
platforms: opts.platforms,

3
react-packager/src/Resolver/index.js vendored
View File

@ -23,7 +23,6 @@ import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type {Reporter} from '../lib/reporting';
import type {TransformCode} from '../node-haste/Module';
import type Cache from '../node-haste/Cache';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
type MinifyCode = (filePath: string, code: string, map: SourceMap) =>
Promise<{code: string, map: SourceMap}>;
@ -33,7 +32,6 @@ type Options = {
blacklistRE?: RegExp,
cache: Cache,
extraNodeModules?: {},
globalTransformCache: ?GlobalTransformCache,
minifyCode: MinifyCode,
platforms: Array<string>,
polyfillModuleNames?: Array<string>,
@ -58,7 +56,6 @@ class Resolver {
assetExts: opts.assetExts,
cache: opts.cache,
extraNodeModules: opts.extraNodeModules,
globalTransformCache: opts.globalTransformCache,
ignoreFilePath: function(filepath) {
return filepath.indexOf('__tests__') !== -1 ||
(opts.blacklistRE != null && opts.blacklistRE.test(filepath));

4
react-packager/src/Server/index.js vendored
View File

@ -33,7 +33,6 @@ import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionRes
import type Bundle from '../Bundler/Bundle';
import type {Reporter} from '../lib/reporting';
import type {GetTransformOptions} from '../Bundler';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
const {
createActionStartEntry,
@ -60,7 +59,6 @@ type Options = {
cacheVersion?: string,
extraNodeModules?: {},
getTransformOptions?: GetTransformOptions,
globalTransformCache: ?GlobalTransformCache,
moduleFormat?: string,
platforms?: Array<string>,
polyfillModuleNames?: Array<string>,
@ -209,7 +207,6 @@ class Server {
cacheVersion: options.cacheVersion || '1.0',
extraNodeModules: options.extraNodeModules || {},
getTransformOptions: options.getTransformOptions,
globalTransformCache: options.globalTransformCache,
moduleFormat: options.moduleFormat != null ? options.moduleFormat : 'haste',
platforms: options.platforms || defaults.platforms,
polyfillModuleNames: options.polyfillModuleNames || [],
@ -239,7 +236,6 @@ class Server {
const bundlerOpts = Object.create(this._opts);
bundlerOpts.assetServer = this._assetServer;
bundlerOpts.allowBundleUpdates = this._opts.watch;
bundlerOpts.globalTransformCache = options.globalTransformCache;
bundlerOpts.watch = this._opts.watch;
bundlerOpts.reporter = options.reporter;
this._bundler = new Bundler(bundlerOpts);

104
react-packager/src/lib/BatchProcessor.js vendored
View File

@ -1,104 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const invariant = require('fbjs/lib/invariant');
type ProcessBatch<TItem, TResult> = (
batch: Array<TItem>,
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
) => mixed;
type BatchProcessorOptions = {
maximumDelayMs: number,
maximumItems: number,
concurrency: number,
};
/**
* We batch items together trying to minimize their processing, for example as
* network queries. For that we wait a small moment before processing a batch.
* We limit also the number of items we try to process in a single batch so that
* if we have many items pending in a short amount of time, we can start
* processing right away.
*/
class BatchProcessor<TItem, TResult> {
_options: BatchProcessorOptions;
_processBatch: ProcessBatch<TItem, TResult>;
_queue: Array<{
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
}>;
_timeoutHandle: ?number;
_currentProcessCount: number;
constructor(
options: BatchProcessorOptions,
processBatch: ProcessBatch<TItem, TResult>,
) {
this._options = options;
this._processBatch = processBatch;
this._queue = [];
this._timeoutHandle = null;
this._currentProcessCount = 0;
(this: any)._processQueue = this._processQueue.bind(this);
}
_processQueue() {
this._timeoutHandle = null;
while (
this._queue.length > 0 &&
this._currentProcessCount < this._options.concurrency
) {
this._currentProcessCount++;
const jobs = this._queue.splice(0, this._options.maximumItems);
const items = jobs.map(job => job.item);
this._processBatch(items, (error, results) => {
invariant(
results == null || results.length === items.length,
'Not enough results returned.',
);
for (let i = 0; i < items.length; ++i) {
jobs[i].callback(error, results && results[i]);
}
this._currentProcessCount--;
this._processQueueOnceReady();
});
}
}
_processQueueOnceReady() {
if (this._queue.length >= this._options.maximumItems) {
clearTimeout(this._timeoutHandle);
process.nextTick(this._processQueue);
return;
}
if (this._timeoutHandle == null) {
this._timeoutHandle = setTimeout(
this._processQueue,
this._options.maximumDelayMs,
);
}
}
queue(
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
) {
this._queue.push({item, callback});
this._processQueueOnceReady();
}
}
module.exports = BatchProcessor;

188
react-packager/src/lib/GlobalTransformCache.js vendored
View File

@ -11,17 +11,15 @@
'use strict';
const BatchProcessor = require('./BatchProcessor');
const crypto = require('crypto');
const imurmurhash = require('imurmurhash');
const invariant = require('fbjs/lib/invariant');
const jsonStableStringify = require('json-stable-stringify');
const path = require('path');
const request = require('request');
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type {CachedResult} from './TransformCache';
import type {Reporter} from './reporting';
type FetchResultURIs = (
keys: Array<string>,
@ -40,9 +38,95 @@ type FetchProps = {
transformOptions: TransformOptions,
};
type FetchCallback = (error?: Error, result?: ?CachedResult) => mixed;
type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
type ProcessBatch<TItem, TResult> = (
batch: Array<TItem>,
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
) => mixed;
type BatchProcessorOptions = {
maximumDelayMs: number,
maximumItems: number,
concurrency: number,
};
/**
* We batch keys together trying to make a smaller amount of queries. For that
* we wait a small moment before starting to fetch. We limit also the number of
* keys we try to fetch at once, so if we already have that many keys pending,
* we can start fetching right away.
*/
class BatchProcessor<TItem, TResult> {
_options: BatchProcessorOptions;
_processBatch: ProcessBatch<TItem, TResult>;
_queue: Array<{
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
}>;
_timeoutHandle: ?number;
_currentProcessCount: number;
constructor(
options: BatchProcessorOptions,
processBatch: ProcessBatch<TItem, TResult>,
) {
this._options = options;
this._processBatch = processBatch;
this._queue = [];
this._timeoutHandle = null;
this._currentProcessCount = 0;
(this: any)._processQueue = this._processQueue.bind(this);
}
_processQueue() {
this._timeoutHandle = null;
while (
this._queue.length > 0 &&
this._currentProcessCount < this._options.concurrency
) {
this._currentProcessCount++;
const jobs = this._queue.splice(0, this._options.maximumItems);
const items = jobs.map(job => job.item);
this._processBatch(items, (error, results) => {
invariant(
results == null || results.length === items.length,
'Not enough results returned.',
);
for (let i = 0; i < items.length; ++i) {
jobs[i].callback(error, results && results[i]);
}
this._currentProcessCount--;
this._processQueueOnceReady();
});
}
}
_processQueueOnceReady() {
if (this._queue.length >= this._options.maximumItems) {
clearTimeout(this._timeoutHandle);
process.nextTick(this._processQueue);
return;
}
if (this._timeoutHandle == null) {
this._timeoutHandle = setTimeout(
this._processQueue,
this._options.maximumDelayMs,
);
}
}
queue(
item: TItem,
callback: (error?: Error, result?: TResult) => mixed,
) {
this._queue.push({item, callback});
this._processQueueOnceReady();
}
}
type URI = string;
/**
@ -51,25 +135,16 @@ type URI = string;
*/
class KeyURIFetcher {
_batchProcessor: BatchProcessor<string, ?URI>;
_fetchResultURIs: FetchResultURIs;
_processError: (error: Error) => mixed;
_batchProcessor: BatchProcessor<string, ?URI>;
/**
* When a batch request fails for some reason, we process the error locally
* and we proceed as if there were no result for these keys instead. That way
* a build will not fail just because of the cache.
*/
_processKeys(
keys: Array<string>,
callback: (error?: Error, keyURIs: Array<?URI>) => mixed,
) {
this._fetchResultURIs(keys, (error, URIsByKey) => {
if (error != null) {
this._processError(error);
}
const URIs = keys.map(key => URIsByKey && URIsByKey.get(key));
callback(undefined, URIs);
callback(error, URIs);
});
}
@ -77,14 +152,13 @@ class KeyURIFetcher {
this._batchProcessor.queue(key, callback);
}
constructor(fetchResultURIs: FetchResultURIs, processError: (error: Error) => mixed) {
constructor(fetchResultURIs: FetchResultURIs) {
this._fetchResultURIs = fetchResultURIs;
this._batchProcessor = new BatchProcessor({
maximumDelayMs: 10,
maximumItems: 500,
concurrency: 25,
}, this._processKeys.bind(this));
this._processError = processError;
}
}
@ -183,46 +257,24 @@ class TransformProfileSet {
}
}
/**
* One can enable the global cache by calling configure() from a custom CLI
* script. Eventually we may make it more flexible.
*/
class GlobalTransformCache {
_fetcher: KeyURIFetcher;
_profileSet: TransformProfileSet;
_reporter: Reporter;
_retries: number;
_store: ?KeyResultStore;
_profileSet: TransformProfileSet;
static _global: ?GlobalTransformCache;
/**
* If too many errors already happened, we just drop the additional errors.
*/
_processError(error: Error) {
if (this._retries <= 0) {
return;
}
this._reporter.update({type: 'global_cache_error', error});
--this._retries;
if (this._retries <= 0) {
this._reporter.update({type: 'global_cache_disabled', reason: 'too_many_errors'});
}
}
/**
* For using the global cache one needs to have some kind of central key-value
* store that gets prefilled using keyOf() and the transformed results. The
* fetching function should provide a mapping of keys to URIs. The files
* referred by these URIs contains the transform results. Using URIs instead
* of returning the content directly allows for independent and parallel
* fetching of each result, that may be arbitrarily large JSON blobs.
*/
constructor(
fetchResultURIs: FetchResultURIs,
storeResults: ?StoreResults,
profiles: Iterable<TransformProfile>,
reporter: Reporter,
) {
this._fetcher = new KeyURIFetcher(fetchResultURIs, this._processError.bind(this));
this._profileSet = new TransformProfileSet(profiles);
this._reporter = reporter;
this._retries = 4;
this._fetcher = new KeyURIFetcher(fetchResultURIs);
if (storeResults != null) {
this._store = new KeyResultStore(storeResults);
}
@ -267,22 +319,8 @@ class GlobalTransformCache {
});
}
/**
* Wrap `_fetchFromURI` with error logging, and return an empty result instead
* of errors. This is because the global cache is not critical to the normal
* packager operation.
*/
_tryFetchingFromURI(uri: string, callback: FetchCallback) {
this._fetchFromURI(uri, (error, result) => {
if (error != null) {
this._processError(error);
}
callback(undefined, result);
});
}
fetch(props: FetchProps, callback: FetchCallback) {
if (this._retries <= 0 || !this._profileSet.has(props.transformOptions)) {
if (!this._profileSet.has(props.transformOptions)) {
process.nextTick(callback);
return;
}
@ -294,7 +332,7 @@ class GlobalTransformCache {
callback();
return;
}
this._tryFetchingFromURI(uri, callback);
this._fetchFromURI(uri, callback);
}
});
}
@ -305,6 +343,32 @@ class GlobalTransformCache {
}
}
/**
* For using the global cache one needs to have some kind of central key-value
* store that gets prefilled using keyOf() and the transformed results. The
* fetching function should provide a mapping of keys to URIs. The files
* referred by these URIs contains the transform results. Using URIs instead
* of returning the content directly allows for independent fetching of each
* result.
*/
static configure(
fetchResultURIs: FetchResultURIs,
storeResults: ?StoreResults,
profiles: Iterable<TransformProfile>,
) {
GlobalTransformCache._global = new GlobalTransformCache(
fetchResultURIs,
storeResults,
profiles,
);
}
static get() {
return GlobalTransformCache._global;
}
}
GlobalTransformCache._global = null;
module.exports = GlobalTransformCache;

3
react-packager/src/lib/TerminalReporter.js vendored
View File

@ -152,8 +152,7 @@ class TerminalReporter {
terminal.log(`${DEP_GRAPH_MESSAGE}, done.`);
break;
case 'global_cache_error':
const message = JSON.stringify(event.error.message);
reporting.logWarning(terminal, 'the global cache failed: %s', message);
reporting.logWarning(terminal, 'The global cache failed: %s', event.error.stack);
break;
case 'global_cache_disabled':
this._logCacheDisabled(event.reason);

52
react-packager/src/node-haste/Module.js vendored
View File

@ -11,6 +11,7 @@
'use strict';
const GlobalTransformCache = require('../lib/GlobalTransformCache');
const TransformCache = require('../lib/TransformCache');
const crypto = require('crypto');
@ -23,7 +24,6 @@ const jsonStableStringify = require('json-stable-stringify');
const {join: joinPath, relative: relativePath, extname} = require('path');
import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {SourceMap} from '../lib/SourceMap';
import type {ReadTransformProps} from '../lib/TransformCache';
import type {Reporter} from '../lib/reporting';
@ -51,15 +51,14 @@ export type Options = {
};
export type ConstructorArgs = {
cache: Cache,
depGraphHelpers: DependencyGraphHelpers,
globalTransformCache: ?GlobalTransformCache,
file: string,
moduleCache: ModuleCache,
cache: Cache,
transformCode: ?TransformCode,
transformCacheKey: ?string,
depGraphHelpers: DependencyGraphHelpers,
options: Options,
reporter: Reporter,
transformCacheKey: ?string,
transformCode: ?TransformCode,
};
class Module {
@ -74,22 +73,22 @@ class Module {
_depGraphHelpers: DependencyGraphHelpers;
_options: Options;
_reporter: Reporter;
_globalCache: ?GlobalTransformCache;
_docBlock: Promise<{id?: string, moduleDocBlock: {[key: string]: mixed}}>;
_readSourceCodePromise: Promise<string>;
_readPromises: Map<string, Promise<ReadResult>>;
static _globalCacheRetries: number;
constructor({
cache,
depGraphHelpers,
file,
globalTransformCache,
moduleCache,
options,
reporter,
transformCacheKey,
cache,
transformCode,
transformCacheKey,
depGraphHelpers,
reporter,
options,
}: ConstructorArgs) {
if (!isAbsolutePath(file)) {
throw new Error('Expected file to be absolute path but got ' + file);
@ -109,7 +108,6 @@ class Module {
this._depGraphHelpers = depGraphHelpers;
this._options = options || {};
this._reporter = reporter;
this._globalCache = globalTransformCache;
this._readPromises = new Map();
}
@ -265,18 +263,28 @@ class Module {
cacheProps: ReadTransformProps,
callback: (error: ?Error, result: ?TransformedCode) => void,
) {
const {_globalCache} = this;
if (_globalCache == null) {
const globalCache = GlobalTransformCache.get();
const noMoreRetries = Module._globalCacheRetries <= 0;
if (globalCache == null || noMoreRetries) {
this._transformCodeForCallback(cacheProps, callback);
return;
}
_globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
if (globalCacheError) {
callback(globalCacheError);
return;
globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
if (globalCacheError != null && Module._globalCacheRetries > 0) {
this._reporter.update({
type: 'global_cache_error',
error: globalCacheError,
});
Module._globalCacheRetries--;
if (Module._globalCacheRetries <= 0) {
this._reporter.update({
type: 'global_cache_disabled',
reason: 'too_many_errors',
});
}
}
if (globalCachedResult == null) {
this._transformAndStoreCodeGlobally(cacheProps, _globalCache, callback);
this._transformAndStoreCodeGlobally(cacheProps, globalCache, callback);
return;
}
callback(undefined, globalCachedResult);
@ -367,6 +375,8 @@ class Module {
}
}
Module._globalCacheRetries = 4;
// use weak map to speed up hash creation of known objects
const knownHashes = new WeakMap();
function stableObjectHash(object) {

23
react-packager/src/node-haste/ModuleCache.js vendored
View File

@ -16,11 +16,13 @@ const Module = require('./Module');
const Package = require('./Package');
const Polyfill = require('./Polyfill');
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {Reporter} from '../lib/reporting';
import type Cache from './Cache';
import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers';
import type {TransformCode, Options as ModuleOptions} from './Module';
import type {
TransformCode,
Options as ModuleOptions,
} from './Module';
type GetClosestPackageFn = (filePath: string) => ?string;
@ -30,7 +32,6 @@ class ModuleCache {
_cache: Cache;
_depGraphHelpers: DependencyGraphHelpers;
_getClosestPackage: GetClosestPackageFn;
_globalTransformCache: ?GlobalTransformCache;
_moduleCache: {[filePath: string]: Module};
_moduleOptions: ModuleOptions;
_packageCache: {[filePath: string]: Package};
@ -46,25 +47,22 @@ class ModuleCache {
depGraphHelpers,
extractRequires,
getClosestPackage,
globalTransformCache,
moduleOptions,
reporter,
transformCacheKey,
transformCode,
reporter,
}: {
assetDependencies: Array<string>,
cache: Cache,
depGraphHelpers: DependencyGraphHelpers,
getClosestPackage: GetClosestPackageFn,
globalTransformCache: ?GlobalTransformCache,
moduleOptions: ModuleOptions,
reporter: Reporter,
transformCacheKey: string,
transformCode: TransformCode,
reporter: Reporter,
}, platforms: Set<string>) {
this._assetDependencies = assetDependencies;
this._getClosestPackage = getClosestPackage;
this._globalTransformCache = globalTransformCache;
this._cache = cache;
this._depGraphHelpers = depGraphHelpers;
this._moduleCache = Object.create(null);
@ -80,15 +78,14 @@ class ModuleCache {
getModule(filePath: string) {
if (!this._moduleCache[filePath]) {
this._moduleCache[filePath] = new Module({
cache: this._cache,
depGraphHelpers: this._depGraphHelpers,
file: filePath,
globalTransformCache: this._globalTransformCache,
moduleCache: this,
cache: this._cache,
transformCode: this._transformCode,
transformCacheKey: this._transformCacheKey,
depGraphHelpers: this._depGraphHelpers,
options: this._moduleOptions,
reporter: this._reporter,
transformCacheKey: this._transformCacheKey,
transformCode: this._transformCode,
});
}
return this._moduleCache[filePath];

6
react-packager/src/node-haste/index.js vendored
View File

@ -38,7 +38,6 @@ const {
} = require('../Logger');
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {Reporter} from '../lib/reporting';
import type {
Options as ModuleOptions,
@ -54,7 +53,6 @@ class DependencyGraph {
extensions: Array<string>,
extraNodeModules: ?Object,
forceNodeFilesystemAPI: boolean,
globalTransformCache: ?GlobalTransformCache,
ignoreFilePath: (filePath: string) => boolean,
maxWorkers: ?number,
mocksPattern: mixed,
@ -89,7 +87,6 @@ class DependencyGraph {
extensions,
extraNodeModules,
forceNodeFilesystemAPI,
globalTransformCache,
ignoreFilePath,
maxWorkers,
mocksPattern,
@ -112,7 +109,6 @@ class DependencyGraph {
extensions?: ?Array<string>,
extraNodeModules: ?Object,
forceNodeFilesystemAPI?: boolean,
globalTransformCache: ?GlobalTransformCache,
ignoreFilePath: (filePath: string) => boolean,
maxWorkers?: ?number,
mocksPattern?: mixed,
@ -134,7 +130,6 @@ class DependencyGraph {
extensions: extensions || ['js', 'json'],
extraNodeModules,
forceNodeFilesystemAPI: !!forceNodeFilesystemAPI,
globalTransformCache,
ignoreFilePath: ignoreFilePath || (() => {}),
maxWorkers,
mocksPattern,
@ -191,7 +186,6 @@ class DependencyGraph {
this._moduleCache = new ModuleCache({
cache: this._cache,
globalTransformCache: this._opts.globalTransformCache,
transformCode: this._opts.transformCode,
transformCacheKey: this._opts.transformCacheKey,
depGraphHelpers: this._helpers,