mirror of https://github.com/status-im/metro.git
Backed out changeset 35b573bece59
Reviewed By: davidaurelio Differential Revision: D4461681 fbshipit-source-id: f5c45f713f2fe68443986a2822aa325b01dd5075
This commit is contained in:
parent
795c1d5148
commit
30f915e818
|
@ -16,12 +16,14 @@ const Logger = require('./src/Logger');
|
||||||
const debug = require('debug');
|
const debug = require('debug');
|
||||||
const invariant = require('fbjs/lib/invariant');
|
const invariant = require('fbjs/lib/invariant');
|
||||||
|
|
||||||
|
import type GlobalTransformCache from './src/lib/GlobalTransformCache';
|
||||||
import type {Reporter} from './src/lib/reporting';
|
import type {Reporter} from './src/lib/reporting';
|
||||||
|
|
||||||
exports.createServer = createServer;
|
exports.createServer = createServer;
|
||||||
exports.Logger = Logger;
|
exports.Logger = Logger;
|
||||||
|
|
||||||
type Options = {
|
type Options = {
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
nonPersistent: boolean,
|
nonPersistent: boolean,
|
||||||
projectRoots: Array<string>,
|
projectRoots: Array<string>,
|
||||||
reporter?: Reporter,
|
reporter?: Reporter,
|
||||||
|
@ -29,6 +31,7 @@ type Options = {
|
||||||
};
|
};
|
||||||
|
|
||||||
type StrictOptions = {
|
type StrictOptions = {
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
nonPersistent: boolean,
|
nonPersistent: boolean,
|
||||||
projectRoots: Array<string>,
|
projectRoots: Array<string>,
|
||||||
reporter: Reporter,
|
reporter: Reporter,
|
||||||
|
|
|
@ -39,6 +39,7 @@ import type Module from '../node-haste/Module';
|
||||||
import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse';
|
import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse';
|
||||||
import type {Options as JSTransformerOptions, TransformOptions} from '../JSTransformer/worker/worker';
|
import type {Options as JSTransformerOptions, TransformOptions} from '../JSTransformer/worker/worker';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
|
|
||||||
export type GetTransformOptions = (
|
export type GetTransformOptions = (
|
||||||
mainModuleName: string,
|
mainModuleName: string,
|
||||||
|
@ -70,6 +71,7 @@ type Options = {
|
||||||
cacheVersion: string,
|
cacheVersion: string,
|
||||||
extraNodeModules: {},
|
extraNodeModules: {},
|
||||||
getTransformOptions?: GetTransformOptions,
|
getTransformOptions?: GetTransformOptions,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
moduleFormat: string,
|
moduleFormat: string,
|
||||||
platforms: Array<string>,
|
platforms: Array<string>,
|
||||||
polyfillModuleNames: Array<string>,
|
polyfillModuleNames: Array<string>,
|
||||||
|
@ -150,6 +152,7 @@ class Bundler {
|
||||||
blacklistRE: opts.blacklistRE,
|
blacklistRE: opts.blacklistRE,
|
||||||
cache: this._cache,
|
cache: this._cache,
|
||||||
extraNodeModules: opts.extraNodeModules,
|
extraNodeModules: opts.extraNodeModules,
|
||||||
|
globalTransformCache: opts.globalTransformCache,
|
||||||
minifyCode: this._transformer.minify,
|
minifyCode: this._transformer.minify,
|
||||||
moduleFormat: opts.moduleFormat,
|
moduleFormat: opts.moduleFormat,
|
||||||
platforms: opts.platforms,
|
platforms: opts.platforms,
|
||||||
|
|
|
@ -23,6 +23,7 @@ import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
import type {TransformCode} from '../node-haste/Module';
|
import type {TransformCode} from '../node-haste/Module';
|
||||||
import type Cache from '../node-haste/Cache';
|
import type Cache from '../node-haste/Cache';
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
|
|
||||||
type MinifyCode = (filePath: string, code: string, map: SourceMap) =>
|
type MinifyCode = (filePath: string, code: string, map: SourceMap) =>
|
||||||
Promise<{code: string, map: SourceMap}>;
|
Promise<{code: string, map: SourceMap}>;
|
||||||
|
@ -32,6 +33,7 @@ type Options = {
|
||||||
blacklistRE?: RegExp,
|
blacklistRE?: RegExp,
|
||||||
cache: Cache,
|
cache: Cache,
|
||||||
extraNodeModules?: {},
|
extraNodeModules?: {},
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
minifyCode: MinifyCode,
|
minifyCode: MinifyCode,
|
||||||
platforms: Array<string>,
|
platforms: Array<string>,
|
||||||
polyfillModuleNames?: Array<string>,
|
polyfillModuleNames?: Array<string>,
|
||||||
|
@ -56,6 +58,7 @@ class Resolver {
|
||||||
assetExts: opts.assetExts,
|
assetExts: opts.assetExts,
|
||||||
cache: opts.cache,
|
cache: opts.cache,
|
||||||
extraNodeModules: opts.extraNodeModules,
|
extraNodeModules: opts.extraNodeModules,
|
||||||
|
globalTransformCache: opts.globalTransformCache,
|
||||||
ignoreFilePath: function(filepath) {
|
ignoreFilePath: function(filepath) {
|
||||||
return filepath.indexOf('__tests__') !== -1 ||
|
return filepath.indexOf('__tests__') !== -1 ||
|
||||||
(opts.blacklistRE != null && opts.blacklistRE.test(filepath));
|
(opts.blacklistRE != null && opts.blacklistRE.test(filepath));
|
||||||
|
|
|
@ -33,6 +33,7 @@ import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionRes
|
||||||
import type Bundle from '../Bundler/Bundle';
|
import type Bundle from '../Bundler/Bundle';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
import type {GetTransformOptions} from '../Bundler';
|
import type {GetTransformOptions} from '../Bundler';
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
|
|
||||||
const {
|
const {
|
||||||
createActionStartEntry,
|
createActionStartEntry,
|
||||||
|
@ -59,6 +60,7 @@ type Options = {
|
||||||
cacheVersion?: string,
|
cacheVersion?: string,
|
||||||
extraNodeModules?: {},
|
extraNodeModules?: {},
|
||||||
getTransformOptions?: GetTransformOptions,
|
getTransformOptions?: GetTransformOptions,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
moduleFormat?: string,
|
moduleFormat?: string,
|
||||||
platforms?: Array<string>,
|
platforms?: Array<string>,
|
||||||
polyfillModuleNames?: Array<string>,
|
polyfillModuleNames?: Array<string>,
|
||||||
|
@ -207,6 +209,7 @@ class Server {
|
||||||
cacheVersion: options.cacheVersion || '1.0',
|
cacheVersion: options.cacheVersion || '1.0',
|
||||||
extraNodeModules: options.extraNodeModules || {},
|
extraNodeModules: options.extraNodeModules || {},
|
||||||
getTransformOptions: options.getTransformOptions,
|
getTransformOptions: options.getTransformOptions,
|
||||||
|
globalTransformCache: options.globalTransformCache,
|
||||||
moduleFormat: options.moduleFormat != null ? options.moduleFormat : 'haste',
|
moduleFormat: options.moduleFormat != null ? options.moduleFormat : 'haste',
|
||||||
platforms: options.platforms || defaults.platforms,
|
platforms: options.platforms || defaults.platforms,
|
||||||
polyfillModuleNames: options.polyfillModuleNames || [],
|
polyfillModuleNames: options.polyfillModuleNames || [],
|
||||||
|
@ -236,6 +239,7 @@ class Server {
|
||||||
const bundlerOpts = Object.create(this._opts);
|
const bundlerOpts = Object.create(this._opts);
|
||||||
bundlerOpts.assetServer = this._assetServer;
|
bundlerOpts.assetServer = this._assetServer;
|
||||||
bundlerOpts.allowBundleUpdates = this._opts.watch;
|
bundlerOpts.allowBundleUpdates = this._opts.watch;
|
||||||
|
bundlerOpts.globalTransformCache = options.globalTransformCache;
|
||||||
bundlerOpts.watch = this._opts.watch;
|
bundlerOpts.watch = this._opts.watch;
|
||||||
bundlerOpts.reporter = options.reporter;
|
bundlerOpts.reporter = options.reporter;
|
||||||
this._bundler = new Bundler(bundlerOpts);
|
this._bundler = new Bundler(bundlerOpts);
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
/**
|
||||||
|
* Copyright (c) 2016-present, Facebook, Inc.
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
|
* This source code is licensed under the BSD-style license found in the
|
||||||
|
* LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
* of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
*
|
||||||
|
* @flow
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const invariant = require('fbjs/lib/invariant');
|
||||||
|
|
||||||
|
type ProcessBatch<TItem, TResult> = (
|
||||||
|
batch: Array<TItem>,
|
||||||
|
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
|
||||||
|
) => mixed;
|
||||||
|
|
||||||
|
type BatchProcessorOptions = {
|
||||||
|
maximumDelayMs: number,
|
||||||
|
maximumItems: number,
|
||||||
|
concurrency: number,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* We batch items together trying to minimize their processing, for example as
|
||||||
|
* network queries. For that we wait a small moment before processing a batch.
|
||||||
|
* We limit also the number of items we try to process in a single batch so that
|
||||||
|
* if we have many items pending in a short amount of time, we can start
|
||||||
|
* processing right away.
|
||||||
|
*/
|
||||||
|
class BatchProcessor<TItem, TResult> {
|
||||||
|
|
||||||
|
_options: BatchProcessorOptions;
|
||||||
|
_processBatch: ProcessBatch<TItem, TResult>;
|
||||||
|
_queue: Array<{
|
||||||
|
item: TItem,
|
||||||
|
callback: (error?: Error, result?: TResult) => mixed,
|
||||||
|
}>;
|
||||||
|
_timeoutHandle: ?number;
|
||||||
|
_currentProcessCount: number;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
options: BatchProcessorOptions,
|
||||||
|
processBatch: ProcessBatch<TItem, TResult>,
|
||||||
|
) {
|
||||||
|
this._options = options;
|
||||||
|
this._processBatch = processBatch;
|
||||||
|
this._queue = [];
|
||||||
|
this._timeoutHandle = null;
|
||||||
|
this._currentProcessCount = 0;
|
||||||
|
(this: any)._processQueue = this._processQueue.bind(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
_processQueue() {
|
||||||
|
this._timeoutHandle = null;
|
||||||
|
while (
|
||||||
|
this._queue.length > 0 &&
|
||||||
|
this._currentProcessCount < this._options.concurrency
|
||||||
|
) {
|
||||||
|
this._currentProcessCount++;
|
||||||
|
const jobs = this._queue.splice(0, this._options.maximumItems);
|
||||||
|
const items = jobs.map(job => job.item);
|
||||||
|
this._processBatch(items, (error, results) => {
|
||||||
|
invariant(
|
||||||
|
results == null || results.length === items.length,
|
||||||
|
'Not enough results returned.',
|
||||||
|
);
|
||||||
|
for (let i = 0; i < items.length; ++i) {
|
||||||
|
jobs[i].callback(error, results && results[i]);
|
||||||
|
}
|
||||||
|
this._currentProcessCount--;
|
||||||
|
this._processQueueOnceReady();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_processQueueOnceReady() {
|
||||||
|
if (this._queue.length >= this._options.maximumItems) {
|
||||||
|
clearTimeout(this._timeoutHandle);
|
||||||
|
process.nextTick(this._processQueue);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this._timeoutHandle == null) {
|
||||||
|
this._timeoutHandle = setTimeout(
|
||||||
|
this._processQueue,
|
||||||
|
this._options.maximumDelayMs,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
queue(
|
||||||
|
item: TItem,
|
||||||
|
callback: (error?: Error, result?: TResult) => mixed,
|
||||||
|
) {
|
||||||
|
this._queue.push({item, callback});
|
||||||
|
this._processQueueOnceReady();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = BatchProcessor;
|
|
@ -11,15 +11,17 @@
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
|
const BatchProcessor = require('./BatchProcessor');
|
||||||
|
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const imurmurhash = require('imurmurhash');
|
const imurmurhash = require('imurmurhash');
|
||||||
const invariant = require('fbjs/lib/invariant');
|
|
||||||
const jsonStableStringify = require('json-stable-stringify');
|
const jsonStableStringify = require('json-stable-stringify');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const request = require('request');
|
const request = require('request');
|
||||||
|
|
||||||
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
|
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
|
||||||
import type {CachedResult} from './TransformCache';
|
import type {CachedResult} from './TransformCache';
|
||||||
|
import type {Reporter} from './reporting';
|
||||||
|
|
||||||
type FetchResultURIs = (
|
type FetchResultURIs = (
|
||||||
keys: Array<string>,
|
keys: Array<string>,
|
||||||
|
@ -38,95 +40,9 @@ type FetchProps = {
|
||||||
transformOptions: TransformOptions,
|
transformOptions: TransformOptions,
|
||||||
};
|
};
|
||||||
|
|
||||||
type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
|
type FetchCallback = (error?: Error, result?: ?CachedResult) => mixed;
|
||||||
type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
|
type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
|
||||||
|
|
||||||
type ProcessBatch<TItem, TResult> = (
|
|
||||||
batch: Array<TItem>,
|
|
||||||
callback: (error?: Error, orderedResults?: Array<TResult>) => mixed,
|
|
||||||
) => mixed;
|
|
||||||
type BatchProcessorOptions = {
|
|
||||||
maximumDelayMs: number,
|
|
||||||
maximumItems: number,
|
|
||||||
concurrency: number,
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* We batch keys together trying to make a smaller amount of queries. For that
|
|
||||||
* we wait a small moment before starting to fetch. We limit also the number of
|
|
||||||
* keys we try to fetch at once, so if we already have that many keys pending,
|
|
||||||
* we can start fetching right away.
|
|
||||||
*/
|
|
||||||
class BatchProcessor<TItem, TResult> {
|
|
||||||
|
|
||||||
_options: BatchProcessorOptions;
|
|
||||||
_processBatch: ProcessBatch<TItem, TResult>;
|
|
||||||
_queue: Array<{
|
|
||||||
item: TItem,
|
|
||||||
callback: (error?: Error, result?: TResult) => mixed,
|
|
||||||
}>;
|
|
||||||
_timeoutHandle: ?number;
|
|
||||||
_currentProcessCount: number;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
options: BatchProcessorOptions,
|
|
||||||
processBatch: ProcessBatch<TItem, TResult>,
|
|
||||||
) {
|
|
||||||
this._options = options;
|
|
||||||
this._processBatch = processBatch;
|
|
||||||
this._queue = [];
|
|
||||||
this._timeoutHandle = null;
|
|
||||||
this._currentProcessCount = 0;
|
|
||||||
(this: any)._processQueue = this._processQueue.bind(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
_processQueue() {
|
|
||||||
this._timeoutHandle = null;
|
|
||||||
while (
|
|
||||||
this._queue.length > 0 &&
|
|
||||||
this._currentProcessCount < this._options.concurrency
|
|
||||||
) {
|
|
||||||
this._currentProcessCount++;
|
|
||||||
const jobs = this._queue.splice(0, this._options.maximumItems);
|
|
||||||
const items = jobs.map(job => job.item);
|
|
||||||
this._processBatch(items, (error, results) => {
|
|
||||||
invariant(
|
|
||||||
results == null || results.length === items.length,
|
|
||||||
'Not enough results returned.',
|
|
||||||
);
|
|
||||||
for (let i = 0; i < items.length; ++i) {
|
|
||||||
jobs[i].callback(error, results && results[i]);
|
|
||||||
}
|
|
||||||
this._currentProcessCount--;
|
|
||||||
this._processQueueOnceReady();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_processQueueOnceReady() {
|
|
||||||
if (this._queue.length >= this._options.maximumItems) {
|
|
||||||
clearTimeout(this._timeoutHandle);
|
|
||||||
process.nextTick(this._processQueue);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (this._timeoutHandle == null) {
|
|
||||||
this._timeoutHandle = setTimeout(
|
|
||||||
this._processQueue,
|
|
||||||
this._options.maximumDelayMs,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
queue(
|
|
||||||
item: TItem,
|
|
||||||
callback: (error?: Error, result?: TResult) => mixed,
|
|
||||||
) {
|
|
||||||
this._queue.push({item, callback});
|
|
||||||
this._processQueueOnceReady();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
type URI = string;
|
type URI = string;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -135,16 +51,25 @@ type URI = string;
|
||||||
*/
|
*/
|
||||||
class KeyURIFetcher {
|
class KeyURIFetcher {
|
||||||
|
|
||||||
_fetchResultURIs: FetchResultURIs;
|
|
||||||
_batchProcessor: BatchProcessor<string, ?URI>;
|
_batchProcessor: BatchProcessor<string, ?URI>;
|
||||||
|
_fetchResultURIs: FetchResultURIs;
|
||||||
|
_processError: (error: Error) => mixed;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* When a batch request fails for some reason, we process the error locally
|
||||||
|
* and we proceed as if there were no result for these keys instead. That way
|
||||||
|
* a build will not fail just because of the cache.
|
||||||
|
*/
|
||||||
_processKeys(
|
_processKeys(
|
||||||
keys: Array<string>,
|
keys: Array<string>,
|
||||||
callback: (error?: Error, keyURIs: Array<?URI>) => mixed,
|
callback: (error?: Error, keyURIs: Array<?URI>) => mixed,
|
||||||
) {
|
) {
|
||||||
this._fetchResultURIs(keys, (error, URIsByKey) => {
|
this._fetchResultURIs(keys, (error, URIsByKey) => {
|
||||||
|
if (error != null) {
|
||||||
|
this._processError(error);
|
||||||
|
}
|
||||||
const URIs = keys.map(key => URIsByKey && URIsByKey.get(key));
|
const URIs = keys.map(key => URIsByKey && URIsByKey.get(key));
|
||||||
callback(error, URIs);
|
callback(undefined, URIs);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -152,13 +77,14 @@ class KeyURIFetcher {
|
||||||
this._batchProcessor.queue(key, callback);
|
this._batchProcessor.queue(key, callback);
|
||||||
}
|
}
|
||||||
|
|
||||||
constructor(fetchResultURIs: FetchResultURIs) {
|
constructor(fetchResultURIs: FetchResultURIs, processError: (error: Error) => mixed) {
|
||||||
this._fetchResultURIs = fetchResultURIs;
|
this._fetchResultURIs = fetchResultURIs;
|
||||||
this._batchProcessor = new BatchProcessor({
|
this._batchProcessor = new BatchProcessor({
|
||||||
maximumDelayMs: 10,
|
maximumDelayMs: 10,
|
||||||
maximumItems: 500,
|
maximumItems: 500,
|
||||||
concurrency: 25,
|
concurrency: 25,
|
||||||
}, this._processKeys.bind(this));
|
}, this._processKeys.bind(this));
|
||||||
|
this._processError = processError;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -257,24 +183,46 @@ class TransformProfileSet {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* One can enable the global cache by calling configure() from a custom CLI
|
|
||||||
* script. Eventually we may make it more flexible.
|
|
||||||
*/
|
|
||||||
class GlobalTransformCache {
|
class GlobalTransformCache {
|
||||||
|
|
||||||
_fetcher: KeyURIFetcher;
|
_fetcher: KeyURIFetcher;
|
||||||
_store: ?KeyResultStore;
|
|
||||||
_profileSet: TransformProfileSet;
|
_profileSet: TransformProfileSet;
|
||||||
static _global: ?GlobalTransformCache;
|
_reporter: Reporter;
|
||||||
|
_retries: number;
|
||||||
|
_store: ?KeyResultStore;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If too many errors already happened, we just drop the additional errors.
|
||||||
|
*/
|
||||||
|
_processError(error: Error) {
|
||||||
|
if (this._retries <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this._reporter.update({type: 'global_cache_error', error});
|
||||||
|
--this._retries;
|
||||||
|
if (this._retries <= 0) {
|
||||||
|
this._reporter.update({type: 'global_cache_disabled', reason: 'too_many_errors'});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For using the global cache one needs to have some kind of central key-value
|
||||||
|
* store that gets prefilled using keyOf() and the transformed results. The
|
||||||
|
* fetching function should provide a mapping of keys to URIs. The files
|
||||||
|
* referred by these URIs contains the transform results. Using URIs instead
|
||||||
|
* of returning the content directly allows for independent and parallel
|
||||||
|
* fetching of each result, that may be arbitrarily large JSON blobs.
|
||||||
|
*/
|
||||||
constructor(
|
constructor(
|
||||||
fetchResultURIs: FetchResultURIs,
|
fetchResultURIs: FetchResultURIs,
|
||||||
storeResults: ?StoreResults,
|
storeResults: ?StoreResults,
|
||||||
profiles: Iterable<TransformProfile>,
|
profiles: Iterable<TransformProfile>,
|
||||||
|
reporter: Reporter,
|
||||||
) {
|
) {
|
||||||
|
this._fetcher = new KeyURIFetcher(fetchResultURIs, this._processError.bind(this));
|
||||||
this._profileSet = new TransformProfileSet(profiles);
|
this._profileSet = new TransformProfileSet(profiles);
|
||||||
this._fetcher = new KeyURIFetcher(fetchResultURIs);
|
this._reporter = reporter;
|
||||||
|
this._retries = 4;
|
||||||
if (storeResults != null) {
|
if (storeResults != null) {
|
||||||
this._store = new KeyResultStore(storeResults);
|
this._store = new KeyResultStore(storeResults);
|
||||||
}
|
}
|
||||||
|
@ -319,8 +267,22 @@ class GlobalTransformCache {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap `_fetchFromURI` with error logging, and return an empty result instead
|
||||||
|
* of errors. This is because the global cache is not critical to the normal
|
||||||
|
* packager operation.
|
||||||
|
*/
|
||||||
|
_tryFetchingFromURI(uri: string, callback: FetchCallback) {
|
||||||
|
this._fetchFromURI(uri, (error, result) => {
|
||||||
|
if (error != null) {
|
||||||
|
this._processError(error);
|
||||||
|
}
|
||||||
|
callback(undefined, result);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
fetch(props: FetchProps, callback: FetchCallback) {
|
fetch(props: FetchProps, callback: FetchCallback) {
|
||||||
if (!this._profileSet.has(props.transformOptions)) {
|
if (this._retries <= 0 || !this._profileSet.has(props.transformOptions)) {
|
||||||
process.nextTick(callback);
|
process.nextTick(callback);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -332,7 +294,7 @@ class GlobalTransformCache {
|
||||||
callback();
|
callback();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this._fetchFromURI(uri, callback);
|
this._tryFetchingFromURI(uri, callback);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -343,32 +305,6 @@ class GlobalTransformCache {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* For using the global cache one needs to have some kind of central key-value
|
|
||||||
* store that gets prefilled using keyOf() and the transformed results. The
|
|
||||||
* fetching function should provide a mapping of keys to URIs. The files
|
|
||||||
* referred by these URIs contains the transform results. Using URIs instead
|
|
||||||
* of returning the content directly allows for independent fetching of each
|
|
||||||
* result.
|
|
||||||
*/
|
|
||||||
static configure(
|
|
||||||
fetchResultURIs: FetchResultURIs,
|
|
||||||
storeResults: ?StoreResults,
|
|
||||||
profiles: Iterable<TransformProfile>,
|
|
||||||
) {
|
|
||||||
GlobalTransformCache._global = new GlobalTransformCache(
|
|
||||||
fetchResultURIs,
|
|
||||||
storeResults,
|
|
||||||
profiles,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
static get() {
|
|
||||||
return GlobalTransformCache._global;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
GlobalTransformCache._global = null;
|
|
||||||
|
|
||||||
module.exports = GlobalTransformCache;
|
module.exports = GlobalTransformCache;
|
||||||
|
|
|
@ -152,7 +152,8 @@ class TerminalReporter {
|
||||||
terminal.log(`${DEP_GRAPH_MESSAGE}, done.`);
|
terminal.log(`${DEP_GRAPH_MESSAGE}, done.`);
|
||||||
break;
|
break;
|
||||||
case 'global_cache_error':
|
case 'global_cache_error':
|
||||||
reporting.logWarning(terminal, 'The global cache failed: %s', event.error.stack);
|
const message = JSON.stringify(event.error.message);
|
||||||
|
reporting.logWarning(terminal, 'the global cache failed: %s', message);
|
||||||
break;
|
break;
|
||||||
case 'global_cache_disabled':
|
case 'global_cache_disabled':
|
||||||
this._logCacheDisabled(event.reason);
|
this._logCacheDisabled(event.reason);
|
||||||
|
|
|
@ -11,7 +11,6 @@
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const GlobalTransformCache = require('../lib/GlobalTransformCache');
|
|
||||||
const TransformCache = require('../lib/TransformCache');
|
const TransformCache = require('../lib/TransformCache');
|
||||||
|
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
|
@ -24,6 +23,7 @@ const jsonStableStringify = require('json-stable-stringify');
|
||||||
const {join: joinPath, relative: relativePath, extname} = require('path');
|
const {join: joinPath, relative: relativePath, extname} = require('path');
|
||||||
|
|
||||||
import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker';
|
import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker';
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
import type {SourceMap} from '../lib/SourceMap';
|
import type {SourceMap} from '../lib/SourceMap';
|
||||||
import type {ReadTransformProps} from '../lib/TransformCache';
|
import type {ReadTransformProps} from '../lib/TransformCache';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
|
@ -51,14 +51,15 @@ export type Options = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ConstructorArgs = {
|
export type ConstructorArgs = {
|
||||||
|
cache: Cache,
|
||||||
|
depGraphHelpers: DependencyGraphHelpers,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
file: string,
|
file: string,
|
||||||
moduleCache: ModuleCache,
|
moduleCache: ModuleCache,
|
||||||
cache: Cache,
|
|
||||||
transformCode: ?TransformCode,
|
|
||||||
transformCacheKey: ?string,
|
|
||||||
depGraphHelpers: DependencyGraphHelpers,
|
|
||||||
options: Options,
|
options: Options,
|
||||||
reporter: Reporter,
|
reporter: Reporter,
|
||||||
|
transformCacheKey: ?string,
|
||||||
|
transformCode: ?TransformCode,
|
||||||
};
|
};
|
||||||
|
|
||||||
class Module {
|
class Module {
|
||||||
|
@ -73,22 +74,22 @@ class Module {
|
||||||
_depGraphHelpers: DependencyGraphHelpers;
|
_depGraphHelpers: DependencyGraphHelpers;
|
||||||
_options: Options;
|
_options: Options;
|
||||||
_reporter: Reporter;
|
_reporter: Reporter;
|
||||||
|
_globalCache: ?GlobalTransformCache;
|
||||||
|
|
||||||
_docBlock: Promise<{id?: string, moduleDocBlock: {[key: string]: mixed}}>;
|
_docBlock: Promise<{id?: string, moduleDocBlock: {[key: string]: mixed}}>;
|
||||||
_readSourceCodePromise: Promise<string>;
|
_readSourceCodePromise: Promise<string>;
|
||||||
_readPromises: Map<string, Promise<ReadResult>>;
|
_readPromises: Map<string, Promise<ReadResult>>;
|
||||||
|
|
||||||
static _globalCacheRetries: number;
|
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
file,
|
|
||||||
moduleCache,
|
|
||||||
cache,
|
cache,
|
||||||
transformCode,
|
|
||||||
transformCacheKey,
|
|
||||||
depGraphHelpers,
|
depGraphHelpers,
|
||||||
reporter,
|
file,
|
||||||
|
globalTransformCache,
|
||||||
|
moduleCache,
|
||||||
options,
|
options,
|
||||||
|
reporter,
|
||||||
|
transformCacheKey,
|
||||||
|
transformCode,
|
||||||
}: ConstructorArgs) {
|
}: ConstructorArgs) {
|
||||||
if (!isAbsolutePath(file)) {
|
if (!isAbsolutePath(file)) {
|
||||||
throw new Error('Expected file to be absolute path but got ' + file);
|
throw new Error('Expected file to be absolute path but got ' + file);
|
||||||
|
@ -108,6 +109,7 @@ class Module {
|
||||||
this._depGraphHelpers = depGraphHelpers;
|
this._depGraphHelpers = depGraphHelpers;
|
||||||
this._options = options || {};
|
this._options = options || {};
|
||||||
this._reporter = reporter;
|
this._reporter = reporter;
|
||||||
|
this._globalCache = globalTransformCache;
|
||||||
|
|
||||||
this._readPromises = new Map();
|
this._readPromises = new Map();
|
||||||
}
|
}
|
||||||
|
@ -263,28 +265,18 @@ class Module {
|
||||||
cacheProps: ReadTransformProps,
|
cacheProps: ReadTransformProps,
|
||||||
callback: (error: ?Error, result: ?TransformedCode) => void,
|
callback: (error: ?Error, result: ?TransformedCode) => void,
|
||||||
) {
|
) {
|
||||||
const globalCache = GlobalTransformCache.get();
|
const {_globalCache} = this;
|
||||||
const noMoreRetries = Module._globalCacheRetries <= 0;
|
if (_globalCache == null) {
|
||||||
if (globalCache == null || noMoreRetries) {
|
|
||||||
this._transformCodeForCallback(cacheProps, callback);
|
this._transformCodeForCallback(cacheProps, callback);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
|
_globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
|
||||||
if (globalCacheError != null && Module._globalCacheRetries > 0) {
|
if (globalCacheError) {
|
||||||
this._reporter.update({
|
callback(globalCacheError);
|
||||||
type: 'global_cache_error',
|
return;
|
||||||
error: globalCacheError,
|
|
||||||
});
|
|
||||||
Module._globalCacheRetries--;
|
|
||||||
if (Module._globalCacheRetries <= 0) {
|
|
||||||
this._reporter.update({
|
|
||||||
type: 'global_cache_disabled',
|
|
||||||
reason: 'too_many_errors',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (globalCachedResult == null) {
|
if (globalCachedResult == null) {
|
||||||
this._transformAndStoreCodeGlobally(cacheProps, globalCache, callback);
|
this._transformAndStoreCodeGlobally(cacheProps, _globalCache, callback);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
callback(undefined, globalCachedResult);
|
callback(undefined, globalCachedResult);
|
||||||
|
@ -375,8 +367,6 @@ class Module {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Module._globalCacheRetries = 4;
|
|
||||||
|
|
||||||
// use weak map to speed up hash creation of known objects
|
// use weak map to speed up hash creation of known objects
|
||||||
const knownHashes = new WeakMap();
|
const knownHashes = new WeakMap();
|
||||||
function stableObjectHash(object) {
|
function stableObjectHash(object) {
|
||||||
|
|
|
@ -16,13 +16,11 @@ const Module = require('./Module');
|
||||||
const Package = require('./Package');
|
const Package = require('./Package');
|
||||||
const Polyfill = require('./Polyfill');
|
const Polyfill = require('./Polyfill');
|
||||||
|
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
import type Cache from './Cache';
|
import type Cache from './Cache';
|
||||||
import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers';
|
import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers';
|
||||||
import type {
|
import type {TransformCode, Options as ModuleOptions} from './Module';
|
||||||
TransformCode,
|
|
||||||
Options as ModuleOptions,
|
|
||||||
} from './Module';
|
|
||||||
|
|
||||||
type GetClosestPackageFn = (filePath: string) => ?string;
|
type GetClosestPackageFn = (filePath: string) => ?string;
|
||||||
|
|
||||||
|
@ -32,6 +30,7 @@ class ModuleCache {
|
||||||
_cache: Cache;
|
_cache: Cache;
|
||||||
_depGraphHelpers: DependencyGraphHelpers;
|
_depGraphHelpers: DependencyGraphHelpers;
|
||||||
_getClosestPackage: GetClosestPackageFn;
|
_getClosestPackage: GetClosestPackageFn;
|
||||||
|
_globalTransformCache: ?GlobalTransformCache;
|
||||||
_moduleCache: {[filePath: string]: Module};
|
_moduleCache: {[filePath: string]: Module};
|
||||||
_moduleOptions: ModuleOptions;
|
_moduleOptions: ModuleOptions;
|
||||||
_packageCache: {[filePath: string]: Package};
|
_packageCache: {[filePath: string]: Package};
|
||||||
|
@ -47,22 +46,25 @@ class ModuleCache {
|
||||||
depGraphHelpers,
|
depGraphHelpers,
|
||||||
extractRequires,
|
extractRequires,
|
||||||
getClosestPackage,
|
getClosestPackage,
|
||||||
|
globalTransformCache,
|
||||||
moduleOptions,
|
moduleOptions,
|
||||||
|
reporter,
|
||||||
transformCacheKey,
|
transformCacheKey,
|
||||||
transformCode,
|
transformCode,
|
||||||
reporter,
|
|
||||||
}: {
|
}: {
|
||||||
assetDependencies: Array<string>,
|
assetDependencies: Array<string>,
|
||||||
cache: Cache,
|
cache: Cache,
|
||||||
depGraphHelpers: DependencyGraphHelpers,
|
depGraphHelpers: DependencyGraphHelpers,
|
||||||
getClosestPackage: GetClosestPackageFn,
|
getClosestPackage: GetClosestPackageFn,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
moduleOptions: ModuleOptions,
|
moduleOptions: ModuleOptions,
|
||||||
|
reporter: Reporter,
|
||||||
transformCacheKey: string,
|
transformCacheKey: string,
|
||||||
transformCode: TransformCode,
|
transformCode: TransformCode,
|
||||||
reporter: Reporter,
|
|
||||||
}, platforms: Set<string>) {
|
}, platforms: Set<string>) {
|
||||||
this._assetDependencies = assetDependencies;
|
this._assetDependencies = assetDependencies;
|
||||||
this._getClosestPackage = getClosestPackage;
|
this._getClosestPackage = getClosestPackage;
|
||||||
|
this._globalTransformCache = globalTransformCache;
|
||||||
this._cache = cache;
|
this._cache = cache;
|
||||||
this._depGraphHelpers = depGraphHelpers;
|
this._depGraphHelpers = depGraphHelpers;
|
||||||
this._moduleCache = Object.create(null);
|
this._moduleCache = Object.create(null);
|
||||||
|
@ -78,14 +80,15 @@ class ModuleCache {
|
||||||
getModule(filePath: string) {
|
getModule(filePath: string) {
|
||||||
if (!this._moduleCache[filePath]) {
|
if (!this._moduleCache[filePath]) {
|
||||||
this._moduleCache[filePath] = new Module({
|
this._moduleCache[filePath] = new Module({
|
||||||
file: filePath,
|
|
||||||
moduleCache: this,
|
|
||||||
cache: this._cache,
|
cache: this._cache,
|
||||||
transformCode: this._transformCode,
|
|
||||||
transformCacheKey: this._transformCacheKey,
|
|
||||||
depGraphHelpers: this._depGraphHelpers,
|
depGraphHelpers: this._depGraphHelpers,
|
||||||
|
file: filePath,
|
||||||
|
globalTransformCache: this._globalTransformCache,
|
||||||
|
moduleCache: this,
|
||||||
options: this._moduleOptions,
|
options: this._moduleOptions,
|
||||||
reporter: this._reporter,
|
reporter: this._reporter,
|
||||||
|
transformCacheKey: this._transformCacheKey,
|
||||||
|
transformCode: this._transformCode,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return this._moduleCache[filePath];
|
return this._moduleCache[filePath];
|
||||||
|
|
|
@ -38,6 +38,7 @@ const {
|
||||||
} = require('../Logger');
|
} = require('../Logger');
|
||||||
|
|
||||||
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
|
import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
|
||||||
|
import type GlobalTransformCache from '../lib/GlobalTransformCache';
|
||||||
import type {Reporter} from '../lib/reporting';
|
import type {Reporter} from '../lib/reporting';
|
||||||
import type {
|
import type {
|
||||||
Options as ModuleOptions,
|
Options as ModuleOptions,
|
||||||
|
@ -53,6 +54,7 @@ class DependencyGraph {
|
||||||
extensions: Array<string>,
|
extensions: Array<string>,
|
||||||
extraNodeModules: ?Object,
|
extraNodeModules: ?Object,
|
||||||
forceNodeFilesystemAPI: boolean,
|
forceNodeFilesystemAPI: boolean,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
ignoreFilePath: (filePath: string) => boolean,
|
ignoreFilePath: (filePath: string) => boolean,
|
||||||
maxWorkers: ?number,
|
maxWorkers: ?number,
|
||||||
mocksPattern: mixed,
|
mocksPattern: mixed,
|
||||||
|
@ -87,6 +89,7 @@ class DependencyGraph {
|
||||||
extensions,
|
extensions,
|
||||||
extraNodeModules,
|
extraNodeModules,
|
||||||
forceNodeFilesystemAPI,
|
forceNodeFilesystemAPI,
|
||||||
|
globalTransformCache,
|
||||||
ignoreFilePath,
|
ignoreFilePath,
|
||||||
maxWorkers,
|
maxWorkers,
|
||||||
mocksPattern,
|
mocksPattern,
|
||||||
|
@ -109,6 +112,7 @@ class DependencyGraph {
|
||||||
extensions?: ?Array<string>,
|
extensions?: ?Array<string>,
|
||||||
extraNodeModules: ?Object,
|
extraNodeModules: ?Object,
|
||||||
forceNodeFilesystemAPI?: boolean,
|
forceNodeFilesystemAPI?: boolean,
|
||||||
|
globalTransformCache: ?GlobalTransformCache,
|
||||||
ignoreFilePath: (filePath: string) => boolean,
|
ignoreFilePath: (filePath: string) => boolean,
|
||||||
maxWorkers?: ?number,
|
maxWorkers?: ?number,
|
||||||
mocksPattern?: mixed,
|
mocksPattern?: mixed,
|
||||||
|
@ -130,6 +134,7 @@ class DependencyGraph {
|
||||||
extensions: extensions || ['js', 'json'],
|
extensions: extensions || ['js', 'json'],
|
||||||
extraNodeModules,
|
extraNodeModules,
|
||||||
forceNodeFilesystemAPI: !!forceNodeFilesystemAPI,
|
forceNodeFilesystemAPI: !!forceNodeFilesystemAPI,
|
||||||
|
globalTransformCache,
|
||||||
ignoreFilePath: ignoreFilePath || (() => {}),
|
ignoreFilePath: ignoreFilePath || (() => {}),
|
||||||
maxWorkers,
|
maxWorkers,
|
||||||
mocksPattern,
|
mocksPattern,
|
||||||
|
@ -186,6 +191,7 @@ class DependencyGraph {
|
||||||
|
|
||||||
this._moduleCache = new ModuleCache({
|
this._moduleCache = new ModuleCache({
|
||||||
cache: this._cache,
|
cache: this._cache,
|
||||||
|
globalTransformCache: this._opts.globalTransformCache,
|
||||||
transformCode: this._opts.transformCode,
|
transformCode: this._opts.transformCode,
|
||||||
transformCacheKey: this._opts.transformCacheKey,
|
transformCacheKey: this._opts.transformCacheKey,
|
||||||
depGraphHelpers: this._helpers,
|
depGraphHelpers: this._helpers,
|
||||||
|
|
Loading…
Reference in New Issue