diff --git a/react-packager/react-packager.js b/react-packager/react-packager.js index 86e5b929..588a7525 100644 --- a/react-packager/react-packager.js +++ b/react-packager/react-packager.js @@ -16,12 +16,14 @@ const Logger = require('./src/Logger'); const debug = require('debug'); const invariant = require('fbjs/lib/invariant'); +import type GlobalTransformCache from './src/lib/GlobalTransformCache'; import type {Reporter} from './src/lib/reporting'; exports.createServer = createServer; exports.Logger = Logger; type Options = { + globalTransformCache: ?GlobalTransformCache, nonPersistent: boolean, projectRoots: Array, reporter?: Reporter, @@ -29,6 +31,7 @@ type Options = { }; type StrictOptions = { + globalTransformCache: ?GlobalTransformCache, nonPersistent: boolean, projectRoots: Array, reporter: Reporter, diff --git a/react-packager/src/Bundler/index.js b/react-packager/src/Bundler/index.js index 221ae418..dc701656 100644 --- a/react-packager/src/Bundler/index.js +++ b/react-packager/src/Bundler/index.js @@ -39,6 +39,7 @@ import type Module from '../node-haste/Module'; import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionResponse'; import type {Options as JSTransformerOptions, TransformOptions} from '../JSTransformer/worker/worker'; import type {Reporter} from '../lib/reporting'; +import type GlobalTransformCache from '../lib/GlobalTransformCache'; export type GetTransformOptions = ( mainModuleName: string, @@ -70,6 +71,7 @@ type Options = { cacheVersion: string, extraNodeModules: {}, getTransformOptions?: GetTransformOptions, + globalTransformCache: ?GlobalTransformCache, moduleFormat: string, platforms: Array, polyfillModuleNames: Array, @@ -150,6 +152,7 @@ class Bundler { blacklistRE: opts.blacklistRE, cache: this._cache, extraNodeModules: opts.extraNodeModules, + globalTransformCache: opts.globalTransformCache, minifyCode: this._transformer.minify, moduleFormat: opts.moduleFormat, platforms: opts.platforms, diff --git a/react-packager/src/Resolver/index.js b/react-packager/src/Resolver/index.js index a32b2fe8..e5f5a839 100644 --- a/react-packager/src/Resolver/index.js +++ b/react-packager/src/Resolver/index.js @@ -23,6 +23,7 @@ import type {Options as TransformOptions} from '../JSTransformer/worker/worker'; import type {Reporter} from '../lib/reporting'; import type {TransformCode} from '../node-haste/Module'; import type Cache from '../node-haste/Cache'; +import type GlobalTransformCache from '../lib/GlobalTransformCache'; type MinifyCode = (filePath: string, code: string, map: SourceMap) => Promise<{code: string, map: SourceMap}>; @@ -32,6 +33,7 @@ type Options = { blacklistRE?: RegExp, cache: Cache, extraNodeModules?: {}, + globalTransformCache: ?GlobalTransformCache, minifyCode: MinifyCode, platforms: Array, polyfillModuleNames?: Array, @@ -56,6 +58,7 @@ class Resolver { assetExts: opts.assetExts, cache: opts.cache, extraNodeModules: opts.extraNodeModules, + globalTransformCache: opts.globalTransformCache, ignoreFilePath: function(filepath) { return filepath.indexOf('__tests__') !== -1 || (opts.blacklistRE != null && opts.blacklistRE.test(filepath)); diff --git a/react-packager/src/Server/index.js b/react-packager/src/Server/index.js index a87cb9b3..811b0357 100644 --- a/react-packager/src/Server/index.js +++ b/react-packager/src/Server/index.js @@ -33,6 +33,7 @@ import type ResolutionResponse from '../node-haste/DependencyGraph/ResolutionRes import type Bundle from '../Bundler/Bundle'; import type {Reporter} from '../lib/reporting'; import type {GetTransformOptions} from '../Bundler'; +import type GlobalTransformCache from '../lib/GlobalTransformCache'; const { createActionStartEntry, @@ -59,6 +60,7 @@ type Options = { cacheVersion?: string, extraNodeModules?: {}, getTransformOptions?: GetTransformOptions, + globalTransformCache: ?GlobalTransformCache, moduleFormat?: string, platforms?: Array, polyfillModuleNames?: Array, @@ -207,6 +209,7 @@ class Server { cacheVersion: options.cacheVersion || '1.0', extraNodeModules: options.extraNodeModules || {}, getTransformOptions: options.getTransformOptions, + globalTransformCache: options.globalTransformCache, moduleFormat: options.moduleFormat != null ? options.moduleFormat : 'haste', platforms: options.platforms || defaults.platforms, polyfillModuleNames: options.polyfillModuleNames || [], @@ -236,6 +239,7 @@ class Server { const bundlerOpts = Object.create(this._opts); bundlerOpts.assetServer = this._assetServer; bundlerOpts.allowBundleUpdates = this._opts.watch; + bundlerOpts.globalTransformCache = options.globalTransformCache; bundlerOpts.watch = this._opts.watch; bundlerOpts.reporter = options.reporter; this._bundler = new Bundler(bundlerOpts); diff --git a/react-packager/src/lib/BatchProcessor.js b/react-packager/src/lib/BatchProcessor.js new file mode 100644 index 00000000..0464534d --- /dev/null +++ b/react-packager/src/lib/BatchProcessor.js @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ + +'use strict'; + +const invariant = require('fbjs/lib/invariant'); + +type ProcessBatch = ( + batch: Array, + callback: (error?: Error, orderedResults?: Array) => mixed, +) => mixed; + +type BatchProcessorOptions = { + maximumDelayMs: number, + maximumItems: number, + concurrency: number, +}; + +/** + * We batch items together trying to minimize their processing, for example as + * network queries. For that we wait a small moment before processing a batch. + * We limit also the number of items we try to process in a single batch so that + * if we have many items pending in a short amount of time, we can start + * processing right away. + */ +class BatchProcessor { + + _options: BatchProcessorOptions; + _processBatch: ProcessBatch; + _queue: Array<{ + item: TItem, + callback: (error?: Error, result?: TResult) => mixed, + }>; + _timeoutHandle: ?number; + _currentProcessCount: number; + + constructor( + options: BatchProcessorOptions, + processBatch: ProcessBatch, + ) { + this._options = options; + this._processBatch = processBatch; + this._queue = []; + this._timeoutHandle = null; + this._currentProcessCount = 0; + (this: any)._processQueue = this._processQueue.bind(this); + } + + _processQueue() { + this._timeoutHandle = null; + while ( + this._queue.length > 0 && + this._currentProcessCount < this._options.concurrency + ) { + this._currentProcessCount++; + const jobs = this._queue.splice(0, this._options.maximumItems); + const items = jobs.map(job => job.item); + this._processBatch(items, (error, results) => { + invariant( + results == null || results.length === items.length, + 'Not enough results returned.', + ); + for (let i = 0; i < items.length; ++i) { + jobs[i].callback(error, results && results[i]); + } + this._currentProcessCount--; + this._processQueueOnceReady(); + }); + } + } + + _processQueueOnceReady() { + if (this._queue.length >= this._options.maximumItems) { + clearTimeout(this._timeoutHandle); + process.nextTick(this._processQueue); + return; + } + if (this._timeoutHandle == null) { + this._timeoutHandle = setTimeout( + this._processQueue, + this._options.maximumDelayMs, + ); + } + } + + queue( + item: TItem, + callback: (error?: Error, result?: TResult) => mixed, + ) { + this._queue.push({item, callback}); + this._processQueueOnceReady(); + } + +} + +module.exports = BatchProcessor; diff --git a/react-packager/src/lib/GlobalTransformCache.js b/react-packager/src/lib/GlobalTransformCache.js index e9df7757..5775a36d 100644 --- a/react-packager/src/lib/GlobalTransformCache.js +++ b/react-packager/src/lib/GlobalTransformCache.js @@ -11,15 +11,17 @@ 'use strict'; +const BatchProcessor = require('./BatchProcessor'); + const crypto = require('crypto'); const imurmurhash = require('imurmurhash'); -const invariant = require('fbjs/lib/invariant'); const jsonStableStringify = require('json-stable-stringify'); const path = require('path'); const request = require('request'); import type {Options as TransformOptions} from '../JSTransformer/worker/worker'; import type {CachedResult} from './TransformCache'; +import type {Reporter} from './reporting'; type FetchResultURIs = ( keys: Array, @@ -38,95 +40,9 @@ type FetchProps = { transformOptions: TransformOptions, }; -type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed; +type FetchCallback = (error?: Error, result?: ?CachedResult) => mixed; type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed; -type ProcessBatch = ( - batch: Array, - callback: (error?: Error, orderedResults?: Array) => mixed, -) => mixed; -type BatchProcessorOptions = { - maximumDelayMs: number, - maximumItems: number, - concurrency: number, -}; - -/** - * We batch keys together trying to make a smaller amount of queries. For that - * we wait a small moment before starting to fetch. We limit also the number of - * keys we try to fetch at once, so if we already have that many keys pending, - * we can start fetching right away. - */ -class BatchProcessor { - - _options: BatchProcessorOptions; - _processBatch: ProcessBatch; - _queue: Array<{ - item: TItem, - callback: (error?: Error, result?: TResult) => mixed, - }>; - _timeoutHandle: ?number; - _currentProcessCount: number; - - constructor( - options: BatchProcessorOptions, - processBatch: ProcessBatch, - ) { - this._options = options; - this._processBatch = processBatch; - this._queue = []; - this._timeoutHandle = null; - this._currentProcessCount = 0; - (this: any)._processQueue = this._processQueue.bind(this); - } - - _processQueue() { - this._timeoutHandle = null; - while ( - this._queue.length > 0 && - this._currentProcessCount < this._options.concurrency - ) { - this._currentProcessCount++; - const jobs = this._queue.splice(0, this._options.maximumItems); - const items = jobs.map(job => job.item); - this._processBatch(items, (error, results) => { - invariant( - results == null || results.length === items.length, - 'Not enough results returned.', - ); - for (let i = 0; i < items.length; ++i) { - jobs[i].callback(error, results && results[i]); - } - this._currentProcessCount--; - this._processQueueOnceReady(); - }); - } - } - - _processQueueOnceReady() { - if (this._queue.length >= this._options.maximumItems) { - clearTimeout(this._timeoutHandle); - process.nextTick(this._processQueue); - return; - } - if (this._timeoutHandle == null) { - this._timeoutHandle = setTimeout( - this._processQueue, - this._options.maximumDelayMs, - ); - } - } - - queue( - item: TItem, - callback: (error?: Error, result?: TResult) => mixed, - ) { - this._queue.push({item, callback}); - this._processQueueOnceReady(); - } - -} - type URI = string; /** @@ -135,16 +51,25 @@ type URI = string; */ class KeyURIFetcher { - _fetchResultURIs: FetchResultURIs; _batchProcessor: BatchProcessor; + _fetchResultURIs: FetchResultURIs; + _processError: (error: Error) => mixed; + /** + * When a batch request fails for some reason, we process the error locally + * and we proceed as if there were no result for these keys instead. That way + * a build will not fail just because of the cache. + */ _processKeys( keys: Array, callback: (error?: Error, keyURIs: Array) => mixed, ) { this._fetchResultURIs(keys, (error, URIsByKey) => { + if (error != null) { + this._processError(error); + } const URIs = keys.map(key => URIsByKey && URIsByKey.get(key)); - callback(error, URIs); + callback(undefined, URIs); }); } @@ -152,13 +77,14 @@ class KeyURIFetcher { this._batchProcessor.queue(key, callback); } - constructor(fetchResultURIs: FetchResultURIs) { + constructor(fetchResultURIs: FetchResultURIs, processError: (error: Error) => mixed) { this._fetchResultURIs = fetchResultURIs; this._batchProcessor = new BatchProcessor({ maximumDelayMs: 10, maximumItems: 500, concurrency: 25, }, this._processKeys.bind(this)); + this._processError = processError; } } @@ -257,24 +183,46 @@ class TransformProfileSet { } } -/** - * One can enable the global cache by calling configure() from a custom CLI - * script. Eventually we may make it more flexible. - */ class GlobalTransformCache { _fetcher: KeyURIFetcher; - _store: ?KeyResultStore; _profileSet: TransformProfileSet; - static _global: ?GlobalTransformCache; + _reporter: Reporter; + _retries: number; + _store: ?KeyResultStore; + /** + * If too many errors already happened, we just drop the additional errors. + */ + _processError(error: Error) { + if (this._retries <= 0) { + return; + } + this._reporter.update({type: 'global_cache_error', error}); + --this._retries; + if (this._retries <= 0) { + this._reporter.update({type: 'global_cache_disabled', reason: 'too_many_errors'}); + } + } + + /** + * For using the global cache one needs to have some kind of central key-value + * store that gets prefilled using keyOf() and the transformed results. The + * fetching function should provide a mapping of keys to URIs. The files + * referred by these URIs contains the transform results. Using URIs instead + * of returning the content directly allows for independent and parallel + * fetching of each result, that may be arbitrarily large JSON blobs. + */ constructor( fetchResultURIs: FetchResultURIs, storeResults: ?StoreResults, profiles: Iterable, + reporter: Reporter, ) { + this._fetcher = new KeyURIFetcher(fetchResultURIs, this._processError.bind(this)); this._profileSet = new TransformProfileSet(profiles); - this._fetcher = new KeyURIFetcher(fetchResultURIs); + this._reporter = reporter; + this._retries = 4; if (storeResults != null) { this._store = new KeyResultStore(storeResults); } @@ -319,8 +267,22 @@ class GlobalTransformCache { }); } + /** + * Wrap `_fetchFromURI` with error logging, and return an empty result instead + * of errors. This is because the global cache is not critical to the normal + * packager operation. + */ + _tryFetchingFromURI(uri: string, callback: FetchCallback) { + this._fetchFromURI(uri, (error, result) => { + if (error != null) { + this._processError(error); + } + callback(undefined, result); + }); + } + fetch(props: FetchProps, callback: FetchCallback) { - if (!this._profileSet.has(props.transformOptions)) { + if (this._retries <= 0 || !this._profileSet.has(props.transformOptions)) { process.nextTick(callback); return; } @@ -332,7 +294,7 @@ class GlobalTransformCache { callback(); return; } - this._fetchFromURI(uri, callback); + this._tryFetchingFromURI(uri, callback); } }); } @@ -343,32 +305,6 @@ class GlobalTransformCache { } } - /** - * For using the global cache one needs to have some kind of central key-value - * store that gets prefilled using keyOf() and the transformed results. The - * fetching function should provide a mapping of keys to URIs. The files - * referred by these URIs contains the transform results. Using URIs instead - * of returning the content directly allows for independent fetching of each - * result. - */ - static configure( - fetchResultURIs: FetchResultURIs, - storeResults: ?StoreResults, - profiles: Iterable, - ) { - GlobalTransformCache._global = new GlobalTransformCache( - fetchResultURIs, - storeResults, - profiles, - ); - } - - static get() { - return GlobalTransformCache._global; - } - } -GlobalTransformCache._global = null; - module.exports = GlobalTransformCache; diff --git a/react-packager/src/lib/TerminalReporter.js b/react-packager/src/lib/TerminalReporter.js index afee692c..6bf7f378 100644 --- a/react-packager/src/lib/TerminalReporter.js +++ b/react-packager/src/lib/TerminalReporter.js @@ -152,7 +152,8 @@ class TerminalReporter { terminal.log(`${DEP_GRAPH_MESSAGE}, done.`); break; case 'global_cache_error': - reporting.logWarning(terminal, 'The global cache failed: %s', event.error.stack); + const message = JSON.stringify(event.error.message); + reporting.logWarning(terminal, 'the global cache failed: %s', message); break; case 'global_cache_disabled': this._logCacheDisabled(event.reason); diff --git a/react-packager/src/node-haste/Module.js b/react-packager/src/node-haste/Module.js index e612ed15..69945c1d 100644 --- a/react-packager/src/node-haste/Module.js +++ b/react-packager/src/node-haste/Module.js @@ -11,7 +11,6 @@ 'use strict'; -const GlobalTransformCache = require('../lib/GlobalTransformCache'); const TransformCache = require('../lib/TransformCache'); const crypto = require('crypto'); @@ -24,6 +23,7 @@ const jsonStableStringify = require('json-stable-stringify'); const {join: joinPath, relative: relativePath, extname} = require('path'); import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker'; +import type GlobalTransformCache from '../lib/GlobalTransformCache'; import type {SourceMap} from '../lib/SourceMap'; import type {ReadTransformProps} from '../lib/TransformCache'; import type {Reporter} from '../lib/reporting'; @@ -51,14 +51,15 @@ export type Options = { }; export type ConstructorArgs = { + cache: Cache, + depGraphHelpers: DependencyGraphHelpers, + globalTransformCache: ?GlobalTransformCache, file: string, moduleCache: ModuleCache, - cache: Cache, - transformCode: ?TransformCode, - transformCacheKey: ?string, - depGraphHelpers: DependencyGraphHelpers, options: Options, reporter: Reporter, + transformCacheKey: ?string, + transformCode: ?TransformCode, }; class Module { @@ -73,22 +74,22 @@ class Module { _depGraphHelpers: DependencyGraphHelpers; _options: Options; _reporter: Reporter; + _globalCache: ?GlobalTransformCache; _docBlock: Promise<{id?: string, moduleDocBlock: {[key: string]: mixed}}>; _readSourceCodePromise: Promise; _readPromises: Map>; - static _globalCacheRetries: number; - constructor({ - file, - moduleCache, cache, - transformCode, - transformCacheKey, depGraphHelpers, - reporter, + file, + globalTransformCache, + moduleCache, options, + reporter, + transformCacheKey, + transformCode, }: ConstructorArgs) { if (!isAbsolutePath(file)) { throw new Error('Expected file to be absolute path but got ' + file); @@ -108,6 +109,7 @@ class Module { this._depGraphHelpers = depGraphHelpers; this._options = options || {}; this._reporter = reporter; + this._globalCache = globalTransformCache; this._readPromises = new Map(); } @@ -263,28 +265,18 @@ class Module { cacheProps: ReadTransformProps, callback: (error: ?Error, result: ?TransformedCode) => void, ) { - const globalCache = GlobalTransformCache.get(); - const noMoreRetries = Module._globalCacheRetries <= 0; - if (globalCache == null || noMoreRetries) { + const {_globalCache} = this; + if (_globalCache == null) { this._transformCodeForCallback(cacheProps, callback); return; } - globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => { - if (globalCacheError != null && Module._globalCacheRetries > 0) { - this._reporter.update({ - type: 'global_cache_error', - error: globalCacheError, - }); - Module._globalCacheRetries--; - if (Module._globalCacheRetries <= 0) { - this._reporter.update({ - type: 'global_cache_disabled', - reason: 'too_many_errors', - }); - } + _globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => { + if (globalCacheError) { + callback(globalCacheError); + return; } if (globalCachedResult == null) { - this._transformAndStoreCodeGlobally(cacheProps, globalCache, callback); + this._transformAndStoreCodeGlobally(cacheProps, _globalCache, callback); return; } callback(undefined, globalCachedResult); @@ -375,8 +367,6 @@ class Module { } } -Module._globalCacheRetries = 4; - // use weak map to speed up hash creation of known objects const knownHashes = new WeakMap(); function stableObjectHash(object) { diff --git a/react-packager/src/node-haste/ModuleCache.js b/react-packager/src/node-haste/ModuleCache.js index 193dac4d..75ad54b2 100644 --- a/react-packager/src/node-haste/ModuleCache.js +++ b/react-packager/src/node-haste/ModuleCache.js @@ -16,13 +16,11 @@ const Module = require('./Module'); const Package = require('./Package'); const Polyfill = require('./Polyfill'); +import type GlobalTransformCache from '../lib/GlobalTransformCache'; import type {Reporter} from '../lib/reporting'; import type Cache from './Cache'; import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers'; -import type { - TransformCode, - Options as ModuleOptions, -} from './Module'; +import type {TransformCode, Options as ModuleOptions} from './Module'; type GetClosestPackageFn = (filePath: string) => ?string; @@ -32,6 +30,7 @@ class ModuleCache { _cache: Cache; _depGraphHelpers: DependencyGraphHelpers; _getClosestPackage: GetClosestPackageFn; + _globalTransformCache: ?GlobalTransformCache; _moduleCache: {[filePath: string]: Module}; _moduleOptions: ModuleOptions; _packageCache: {[filePath: string]: Package}; @@ -47,22 +46,25 @@ class ModuleCache { depGraphHelpers, extractRequires, getClosestPackage, + globalTransformCache, moduleOptions, + reporter, transformCacheKey, transformCode, - reporter, }: { assetDependencies: Array, cache: Cache, depGraphHelpers: DependencyGraphHelpers, getClosestPackage: GetClosestPackageFn, + globalTransformCache: ?GlobalTransformCache, moduleOptions: ModuleOptions, + reporter: Reporter, transformCacheKey: string, transformCode: TransformCode, - reporter: Reporter, }, platforms: Set) { this._assetDependencies = assetDependencies; this._getClosestPackage = getClosestPackage; + this._globalTransformCache = globalTransformCache; this._cache = cache; this._depGraphHelpers = depGraphHelpers; this._moduleCache = Object.create(null); @@ -78,14 +80,15 @@ class ModuleCache { getModule(filePath: string) { if (!this._moduleCache[filePath]) { this._moduleCache[filePath] = new Module({ - file: filePath, - moduleCache: this, cache: this._cache, - transformCode: this._transformCode, - transformCacheKey: this._transformCacheKey, depGraphHelpers: this._depGraphHelpers, + file: filePath, + globalTransformCache: this._globalTransformCache, + moduleCache: this, options: this._moduleOptions, reporter: this._reporter, + transformCacheKey: this._transformCacheKey, + transformCode: this._transformCode, }); } return this._moduleCache[filePath]; diff --git a/react-packager/src/node-haste/index.js b/react-packager/src/node-haste/index.js index a37f9627..5c64e01c 100644 --- a/react-packager/src/node-haste/index.js +++ b/react-packager/src/node-haste/index.js @@ -38,6 +38,7 @@ const { } = require('../Logger'); import type {Options as TransformOptions} from '../JSTransformer/worker/worker'; +import type GlobalTransformCache from '../lib/GlobalTransformCache'; import type {Reporter} from '../lib/reporting'; import type { Options as ModuleOptions, @@ -53,6 +54,7 @@ class DependencyGraph { extensions: Array, extraNodeModules: ?Object, forceNodeFilesystemAPI: boolean, + globalTransformCache: ?GlobalTransformCache, ignoreFilePath: (filePath: string) => boolean, maxWorkers: ?number, mocksPattern: mixed, @@ -87,6 +89,7 @@ class DependencyGraph { extensions, extraNodeModules, forceNodeFilesystemAPI, + globalTransformCache, ignoreFilePath, maxWorkers, mocksPattern, @@ -109,6 +112,7 @@ class DependencyGraph { extensions?: ?Array, extraNodeModules: ?Object, forceNodeFilesystemAPI?: boolean, + globalTransformCache: ?GlobalTransformCache, ignoreFilePath: (filePath: string) => boolean, maxWorkers?: ?number, mocksPattern?: mixed, @@ -130,6 +134,7 @@ class DependencyGraph { extensions: extensions || ['js', 'json'], extraNodeModules, forceNodeFilesystemAPI: !!forceNodeFilesystemAPI, + globalTransformCache, ignoreFilePath: ignoreFilePath || (() => {}), maxWorkers, mocksPattern, @@ -186,6 +191,7 @@ class DependencyGraph { this._moduleCache = new ModuleCache({ cache: this._cache, + globalTransformCache: this._opts.globalTransformCache, transformCode: this._opts.transformCode, transformCacheKey: this._opts.transformCacheKey, depGraphHelpers: this._helpers,