From 158da01d26c910253366f644eb045ee09251fcd2 Mon Sep 17 00:00:00 2001 From: Jean Lauliac Date: Thu, 24 Nov 2016 09:56:30 -0800 Subject: [PATCH] packager: add GlobalTransformCache Reviewed By: davidaurelio Differential Revision: D4175938 fbshipit-source-id: 1f57d594b4c8c8189feb2ea6d4d4011870ffd85f --- .../src/Server/__tests__/Server-test.js | 3 +- .../src/lib/GlobalTransformCache.js | 198 ++++++++++++++++++ react-packager/src/lib/TransformCache.js | 24 ++- .../src/lib/__mocks__/GlobalTransformCache.js | 18 ++ .../src/lib/__tests__/TransformCache-test.js | 4 +- react-packager/src/lib/toFixedHex.js | 20 ++ react-packager/src/node-haste/Module.js | 103 ++++++--- 7 files changed, 324 insertions(+), 46 deletions(-) create mode 100644 react-packager/src/lib/GlobalTransformCache.js create mode 100644 react-packager/src/lib/__mocks__/GlobalTransformCache.js create mode 100644 react-packager/src/lib/toFixedHex.js diff --git a/react-packager/src/Server/__tests__/Server-test.js b/react-packager/src/Server/__tests__/Server-test.js index 660bac88..57ed011f 100644 --- a/react-packager/src/Server/__tests__/Server-test.js +++ b/react-packager/src/Server/__tests__/Server-test.js @@ -19,7 +19,8 @@ jest.setMock('worker-farm', function() { return () => {}; }) .mock('../../AssetServer') .mock('../../lib/declareOpts') .mock('../../node-haste') - .mock('../../Logger'); + .mock('../../Logger') + .mock('../../lib/GlobalTransformCache'); describe('processRequest', () => { let SourceMapConsumer, Bundler, Server, AssetServer, Promise; diff --git a/react-packager/src/lib/GlobalTransformCache.js b/react-packager/src/lib/GlobalTransformCache.js new file mode 100644 index 00000000..ef86993b --- /dev/null +++ b/react-packager/src/lib/GlobalTransformCache.js @@ -0,0 +1,198 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ + +'use strict'; + +const debounce = require('lodash/debounce'); +const imurmurhash = require('imurmurhash'); +const jsonStableStringify = require('json-stable-stringify'); +const path = require('path'); +const request = require('request'); +const toFixedHex = require('./toFixedHex'); + +import type {CachedResult} from './TransformCache'; + +const SINGLE_REQUEST_MAX_KEYS = 100; +const AGGREGATION_DELAY_MS = 100; + +type FetchResultURIs = ( + keys: Array, + callback: (error?: Error, results?: Map) => void, +) => mixed; + +type FetchProps = { + filePath: string, + sourceCode: string, + transformCacheKey: string, + transformOptions: mixed, +}; + +type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed; +type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed; + +/** + * We aggregate the requests to do a single request for many keys. It also + * ensures we do a single request at a time to avoid pressuring the I/O. + */ +class KeyURIFetcher { + + _fetchResultURIs: FetchResultURIs; + _pendingQueries: Array<{key: string, callback: FetchURICallback}>; + _isProcessing: boolean; + _processQueriesDebounced: () => void; + _processQueries: () => void; + + /** + * Fetch the pending keys right now, if any and if we're not already doing + * so in parallel. At the end of the fetch, we trigger a new batch fetching + * recursively. + */ + _processQueries() { + const {_pendingQueries} = this; + if (_pendingQueries.length === 0 || this._isProcessing) { + return; + } + this._isProcessing = true; + const queries = _pendingQueries.splice(0, SINGLE_REQUEST_MAX_KEYS); + const keys = queries.map(query => query.key); + this._fetchResultURIs(keys, (error, results) => { + queries.forEach(query => { + query.callback(error, results && results.get(query.key)); + }); + this._isProcessing = false; + process.nextTick(this._processQueries); + }); + } + + /** + * Enqueue the fetching of a particular key. + */ + fetch(key: string, callback: FetchURICallback) { + this._pendingQueries.push({key, callback}); + this._processQueriesDebounced(); + } + + constructor(fetchResultURIs: FetchResultURIs) { + this._fetchResultURIs = fetchResultURIs; + this._pendingQueries = []; + this._isProcessing = false; + this._processQueries = this._processQueries.bind(this); + this._processQueriesDebounced = + debounce(this._processQueries, AGGREGATION_DELAY_MS); + } + +} + +function validateCachedResult(cachedResult: mixed): ?CachedResult { + if ( + cachedResult != null && + typeof cachedResult === 'object' && + typeof cachedResult.code === 'string' && + Array.isArray(cachedResult.dependencies) && + cachedResult.dependencies.every(dep => typeof dep === 'string') && + Array.isArray(cachedResult.dependencyOffsets) && + cachedResult.dependencyOffsets.every(offset => typeof offset === 'number') + ) { + return (cachedResult: any); + } + return undefined; +} + +/** + * One can enable the global cache by calling configure() from a custom CLI + * script. Eventually we may make it more flexible. + */ +class GlobalTransformCache { + + _fetcher: KeyURIFetcher; + static _global: ?GlobalTransformCache; + + constructor(fetchResultURIs: FetchResultURIs) { + this._fetcher = new KeyURIFetcher(fetchResultURIs); + } + + /** + * Return a key for identifying uniquely a source file. + */ + static keyOf(props: FetchProps) { + const sourceDigest = toFixedHex(8, imurmurhash(props.sourceCode).result()); + const optionsHash = imurmurhash() + .hash(jsonStableStringify(props.transformOptions) || '') + .hash(props.transformCacheKey) + .result(); + const optionsDigest = toFixedHex(8, optionsHash); + return ( + `${optionsDigest}${sourceDigest}` + + `${path.basename(props.filePath)}` + ); + } + + /** + * We may want to improve that logic to return a stream instead of the whole + * blob of transformed results. However the results are generally only a few + * megabytes each. + */ + _fetchFromURI(uri: string, callback: FetchCallback) { + request.get({uri, json: true}, (error, response, unvalidatedResult) => { + if (error != null) { + callback(error); + return; + } + if (response.statusCode !== 200) { + callback(new Error( + `Unexpected HTTP status code: ${response.statusCode}`, + )); + return; + } + const result = validateCachedResult(unvalidatedResult); + if (result == null) { + callback(new Error('Invalid result returned by server.')); + return; + } + callback(undefined, result); + }); + } + + fetch(props: FetchProps, callback: FetchCallback) { + this._fetcher.fetch(GlobalTransformCache.keyOf(props), (error, uri) => { + if (error != null) { + callback(error); + } else { + if (uri == null) { + callback(); + return; + } + this._fetchFromURI(uri, callback); + } + }); + } + + /** + * For using the global cache one needs to have some kind of central key-value + * store that gets prefilled using keyOf() and the transformed results. The + * fetching function should provide a mapping of keys to URIs. The files + * referred by these URIs contains the transform results. Using URIs instead + * of returning the content directly allows for independent fetching of each + * result. + */ + static configure(fetchResultURIs: FetchResultURIs) { + GlobalTransformCache._global = new GlobalTransformCache(fetchResultURIs); + } + + static get() { + return GlobalTransformCache._global; + } + +} + +GlobalTransformCache._global = null; + +module.exports = GlobalTransformCache; diff --git a/react-packager/src/lib/TransformCache.js b/react-packager/src/lib/TransformCache.js index 0be0e0b9..07168629 100644 --- a/react-packager/src/lib/TransformCache.js +++ b/react-packager/src/lib/TransformCache.js @@ -22,6 +22,7 @@ const jsonStableStringify = require('json-stable-stringify'); const mkdirp = require('mkdirp'); const path = require('path'); const rimraf = require('rimraf'); +const toFixedHex = require('./toFixedHex'); const writeFileAtomicSync = require('write-file-atomic').sync; const CACHE_NAME = 'react-native-packager-cache'; @@ -66,15 +67,14 @@ function getCacheFilePaths(props: { const hasher = imurmurhash() .hash(props.filePath) .hash(jsonStableStringify(props.transformOptions) || ''); - let hash = hasher.result().toString(16); - hash = Array(8 - hash.length + 1).join('0') + hash; + const hash = toFixedHex(8, hasher.result()); const prefix = hash.substr(0, 2); const fileName = `${hash.substr(2)}${path.basename(props.filePath)}`; const base = path.join(getCacheDirPath(), prefix, fileName); return {transformedCode: base, metadata: base + '.meta'}; } -type CachedResult = { +export type CachedResult = { code: string, dependencies: Array, dependencyOffsets: Array, @@ -135,7 +135,7 @@ function writeSync(props: { ])); } -type CacheOptions = {resetCache?: boolean}; +export type CacheOptions = {resetCache?: boolean}; /* 1 day */ const GARBAGE_COLLECTION_PERIOD = 24 * 60 * 60 * 1000; @@ -272,6 +272,14 @@ function readMetadataFileSync( }; } +export type ReadTransformProps = { + filePath: string, + sourceCode: string, + transformOptions: mixed, + transformCacheKey: string, + cacheOptions: CacheOptions, +}; + /** * We verify the source hash matches to ensure we always favor rebuilding when * source change (rather than just using fs.mtime(), a bit less robust). @@ -285,13 +293,7 @@ function readMetadataFileSync( * Meanwhile we store transforms with different options in different files so * that it is fast to switch between ex. minified, or not. */ -function readSync(props: { - filePath: string, - sourceCode: string, - transformOptions: mixed, - transformCacheKey: string, - cacheOptions: CacheOptions, -}): ?CachedResult { +function readSync(props: ReadTransformProps): ?CachedResult { GARBAGE_COLLECTOR.collectIfNecessarySync(props.cacheOptions); const cacheFilePaths = getCacheFilePaths(props); let metadata, transformedCode; diff --git a/react-packager/src/lib/__mocks__/GlobalTransformCache.js b/react-packager/src/lib/__mocks__/GlobalTransformCache.js new file mode 100644 index 00000000..f741d772 --- /dev/null +++ b/react-packager/src/lib/__mocks__/GlobalTransformCache.js @@ -0,0 +1,18 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ + +'use strict'; + +function get() { + return null; +} + +module.exports = {get}; diff --git a/react-packager/src/lib/__tests__/TransformCache-test.js b/react-packager/src/lib/__tests__/TransformCache-test.js index f6a83126..e2e3bece 100644 --- a/react-packager/src/lib/__tests__/TransformCache-test.js +++ b/react-packager/src/lib/__tests__/TransformCache-test.js @@ -12,7 +12,9 @@ jest .dontMock('imurmurhash') .dontMock('json-stable-stringify') - .dontMock('../TransformCache'); + .dontMock('../TransformCache') + .dontMock('../toFixedHex') + .dontMock('left-pad'); const imurmurhash = require('imurmurhash'); diff --git a/react-packager/src/lib/toFixedHex.js b/react-packager/src/lib/toFixedHex.js new file mode 100644 index 00000000..3952e15a --- /dev/null +++ b/react-packager/src/lib/toFixedHex.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ + +'use strict'; + +const leftPad = require('left-pad'); + +function toFixedHex(length: number, number: number): string { + return leftPad(number.toString(16), length, '0'); +} + +module.exports = toFixedHex; diff --git a/react-packager/src/node-haste/Module.js b/react-packager/src/node-haste/Module.js index 1e716dad..c1a2f19d 100644 --- a/react-packager/src/node-haste/Module.js +++ b/react-packager/src/node-haste/Module.js @@ -11,8 +11,10 @@ 'use strict'; +const GlobalTransformCache = require('../lib/GlobalTransformCache'); const TransformCache = require('../lib/TransformCache'); +const chalk = require('chalk'); const crypto = require('crypto'); const docblock = require('./DependencyGraph/docblock'); const invariant = require('invariant'); @@ -22,6 +24,7 @@ const jsonStableStringify = require('json-stable-stringify'); const {join: joinPath, relative: relativePath, extname} = require('path'); import type {TransformedCode} from '../JSTransformer/worker/worker'; +import type {ReadTransformProps} from '../lib/TransformCache'; import type Cache from './Cache'; import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers'; import type ModuleCache from './ModuleCache'; @@ -73,6 +76,8 @@ class Module { _readSourceCodePromise: Promise; _readPromises: Map>; + static _useGlobalCache: boolean; + constructor({ file, fastfs, @@ -214,31 +219,61 @@ class Module { return {...result, id, source}; } - _transformAndCache( - transformOptions: mixed, + _transformCodeForCallback( + cacheProps: ReadTransformProps, callback: (error: ?Error, result: ?TransformedCode) => void, ) { - const {_transformCode, _transformCacheKey} = this; + const {_transformCode} = this; invariant(_transformCode != null, 'missing code transform funtion'); - invariant(_transformCacheKey != null, 'missing cache key'); - this._readSourceCode() - .then(sourceCode => - _transformCode(this, sourceCode, transformOptions) - .then(freshResult => { - TransformCache.writeSync({ - filePath: this.path, - sourceCode, - transformCacheKey: _transformCacheKey, - transformOptions, - result: freshResult, - }); - return freshResult; - }) - ) - .then( - freshResult => process.nextTick(callback, null, freshResult), - error => process.nextTick(callback, error), - ); + const {sourceCode, transformOptions} = cacheProps; + return _transformCode(this, sourceCode, transformOptions).then( + freshResult => process.nextTick(callback, undefined, freshResult), + error => process.nextTick(callback, error), + ); + } + + _getTransformedCode( + cacheProps: ReadTransformProps, + callback: (error: ?Error, result: ?TransformedCode) => void, + ) { + const globalCache = GlobalTransformCache.get(); + if (!Module._useGlobalCache || globalCache == null) { + this._transformCodeForCallback(cacheProps, callback); + return; + } + globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => { + if (globalCacheError != null && Module._useGlobalCache) { + console.log(chalk.red( + '\nWarning: the global cache failed with error:', + )); + console.log(chalk.red(globalCacheError.stack)); + console.log(chalk.red( + 'The global cache will be DISABLED for the ' + + 'remainder of the transformation.', + )); + Module._useGlobalCache = false; + } + if (globalCacheError != null || globalCachedResult == null) { + this._transformCodeForCallback(cacheProps, callback); + return; + } + callback(undefined, globalCachedResult); + }); + } + + _getAndCacheTransformedCode( + cacheProps: ReadTransformProps, + callback: (error: ?Error, result: ?TransformedCode) => void, + ) { + this._getTransformedCode(cacheProps, (error, result) => { + if (error) { + callback(error); + return; + } + invariant(result != null, 'missing result'); + TransformCache.writeSync({...cacheProps, result}); + callback(undefined, result); + }); } /** @@ -264,20 +299,20 @@ class Module { } const transformCacheKey = this._transformCacheKey; invariant(transformCacheKey != null, 'missing transform cache key'); - const cachedResult = - TransformCache.readSync({ - filePath: this.path, - sourceCode, - transformCacheKey, - transformOptions, - cacheOptions: this._options, - }); + const cacheProps = { + filePath: this.path, + sourceCode, + transformCacheKey, + transformOptions, + cacheOptions: this._options, + }; + const cachedResult = TransformCache.readSync(cacheProps); if (cachedResult) { - return this._finalizeReadResult(sourceCode, id, extern, cachedResult); + return Promise.resolve(this._finalizeReadResult(sourceCode, id, extern, cachedResult)); } return new Promise((resolve, reject) => { - this._transformAndCache( - transformOptions, + this._getAndCacheTransformedCode( + cacheProps, (transformError, freshResult) => { if (transformError) { reject(transformError); @@ -320,6 +355,8 @@ class Module { } } +Module._useGlobalCache = true; + // use weak map to speed up hash creation of known objects const knownHashes = new WeakMap(); function stableObjectHash(object) {