From d1ea86b84cb18426dbb4fed21271ec9d4942de1a Mon Sep 17 00:00:00 2001 From: Jean Lauliac Date: Thu, 6 Apr 2017 04:37:31 -0700 Subject: [PATCH] packager: GlobalTransformCache: make key globalized Reviewed By: davidaurelio Differential Revision: D4835217 fbshipit-source-id: b43456e1e1f83c849a887b07f4f01f8ed0e9df4b --- packages/metro-bundler/react-packager.js | 2 +- packages/metro-bundler/src/Bundler/index.js | 10 +- .../src/JSTransformer/worker/worker.js | 6 +- packages/metro-bundler/src/Resolver/index.js | 2 +- packages/metro-bundler/src/Server/index.js | 2 +- .../src/lib/GlobalTransformCache.js | 194 +++++++++++++----- .../__tests__/GlobalTransformCache-test.js | 30 ++- .../GlobalTransformCache-test.js.snap | 4 +- .../metro-bundler/src/node-haste/Module.js | 2 +- .../src/node-haste/ModuleCache.js | 2 +- .../metro-bundler/src/node-haste/index.js | 2 +- 11 files changed, 187 insertions(+), 69 deletions(-) diff --git a/packages/metro-bundler/react-packager.js b/packages/metro-bundler/react-packager.js index e5159230..b66418c6 100644 --- a/packages/metro-bundler/react-packager.js +++ b/packages/metro-bundler/react-packager.js @@ -17,7 +17,7 @@ const debug = require('debug'); const invariant = require('fbjs/lib/invariant'); import type Server from './src/Server'; -import type GlobalTransformCache from './src/lib/GlobalTransformCache'; +import type {GlobalTransformCache} from './src/lib/GlobalTransformCache'; import type {Reporter} from './src/lib/reporting'; import type {HasteImpl} from './src/node-haste/Module'; diff --git a/packages/metro-bundler/src/Bundler/index.js b/packages/metro-bundler/src/Bundler/index.js index 8197ff91..e2d8fc8f 100644 --- a/packages/metro-bundler/src/Bundler/index.js +++ b/packages/metro-bundler/src/Bundler/index.js @@ -46,13 +46,19 @@ import type { TransformOptions, } from '../JSTransformer/worker/worker'; import type {Reporter} from '../lib/reporting'; -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; + +export type ExtraTransformOptions = { + +inlineRequires?: {+blacklist: {[string]: true}} | boolean, + +preloadedModules?: Array | false, + +ramGroups?: Array, +}; export type GetTransformOptions = ( mainModuleName: string, options: {}, getDependencies: string => Promise>, -) => {} | Promise<{}>; +) => ExtraTransformOptions | Promise; type Asset = { __packager_asset: boolean, diff --git a/packages/metro-bundler/src/JSTransformer/worker/worker.js b/packages/metro-bundler/src/JSTransformer/worker/worker.js index acd73726..e77614a2 100644 --- a/packages/metro-bundler/src/JSTransformer/worker/worker.js +++ b/packages/metro-bundler/src/JSTransformer/worker/worker.js @@ -36,15 +36,19 @@ type Transformer = { }; export type TransformOptions = { + +dev: boolean, generateSourceMaps: boolean, + +hot: boolean, + +inlineRequires: {+blacklist: {[string]: true}} | boolean, platform: string, - preloadedModules?: Array, + preloadedModules?: Array | false, projectRoots: Array, ramGroups?: Array, } & BabelTransformOptions; export type Options = { +dev: boolean, + +extern?: boolean, +minify: boolean, platform: string, transform: TransformOptions, diff --git a/packages/metro-bundler/src/Resolver/index.js b/packages/metro-bundler/src/Resolver/index.js index e564706d..3538524b 100644 --- a/packages/metro-bundler/src/Resolver/index.js +++ b/packages/metro-bundler/src/Resolver/index.js @@ -24,7 +24,7 @@ import type {Reporter} from '../lib/reporting'; import type {TransformCode} from '../node-haste/Module'; import type Cache from '../node-haste/Cache'; import type {GetTransformCacheKey} from '../lib/TransformCache'; -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; type MinifyCode = (filePath: string, code: string, map: SourceMap) => Promise<{code: string, map: SourceMap}>; diff --git a/packages/metro-bundler/src/Server/index.js b/packages/metro-bundler/src/Server/index.js index d7710ad2..a5df8a8b 100644 --- a/packages/metro-bundler/src/Server/index.js +++ b/packages/metro-bundler/src/Server/index.js @@ -34,7 +34,7 @@ import type Bundle from '../Bundler/Bundle'; import type HMRBundle from '../Bundler/HMRBundle'; import type {Reporter} from '../lib/reporting'; import type {GetTransformOptions} from '../Bundler'; -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; import type {SourceMap, Symbolicate} from './symbolicate'; const { diff --git a/packages/metro-bundler/src/lib/GlobalTransformCache.js b/packages/metro-bundler/src/lib/GlobalTransformCache.js index 13de60eb..962c028d 100644 --- a/packages/metro-bundler/src/lib/GlobalTransformCache.js +++ b/packages/metro-bundler/src/lib/GlobalTransformCache.js @@ -16,22 +16,53 @@ const FetchError = require('node-fetch/lib/fetch-error'); const crypto = require('crypto'); const fetch = require('node-fetch'); -const imurmurhash = require('imurmurhash'); const jsonStableStringify = require('json-stable-stringify'); const path = require('path'); -import type {Options as TransformOptions} from '../JSTransformer/worker/worker'; +import type { + Options as TransformWorkerOptions, + TransformOptions, +} from '../JSTransformer/worker/worker'; import type {CachedResult, GetTransformCacheKey} from './TransformCache'; +/** + * The API that a global transform cache must comply with. To implement a + * custom cache, implement this interface and pass it as argument to the + * application's top-level `Server` class. + */ +export type GlobalTransformCache = { + /** + * Synchronously determine if it is worth trying to fetch a result from the + * cache. This can be used, for instance, to exclude sets of options we know + * will never be cached. + */ + shouldFetch(props: FetchProps): boolean, + + /** + * Try to fetch a result. It doesn't actually need to fetch from a server, + * the global cache could be instantiated locally for example. + */ + fetch(props: FetchProps): Promise, + + /** + * Try to store a result, without waiting for the success or failure of the + * operation. Consequently, the actual storage operation could be done at a + * much later point if desired. It is recommended to actually have this + * function be a no-op in production, and only do the storage operation from + * a script running on your Continuous Integration platform. + */ + store(props: FetchProps, result: CachedResult): void, +}; + type FetchResultURIs = (keys: Array) => Promise>; type FetchResultFromURI = (uri: string) => Promise; type StoreResults = (resultsByKey: Map) => Promise; -type FetchProps = { +export type FetchProps = { filePath: string, sourceCode: string, getTransformCacheKey: GetTransformCacheKey, - transformOptions: TransformOptions, + transformOptions: TransformWorkerOptions, }; type URI = string; @@ -98,29 +129,6 @@ class KeyResultStore { } -/** - * The transform options contain absolute paths. This can contain, for example, - * the username if someone works their home directory (very likely). We get rid - * of this local data for the global cache, otherwise nobody would share the - * same cache keys. The project roots should not be needed as part of the cache - * key as they should not affect the transformation of a single particular file. - */ -function globalizeTransformOptions( - options: TransformOptions, -): TransformOptions { - const {transform} = options; - if (transform == null) { - return options; - } - return { - ...options, - transform: { - ...transform, - projectRoots: [], - }, - }; -} - export type TransformProfile = {+dev: boolean, +minify: boolean, +platform: string}; function profileKey({dev, minify, platform}: TransformProfile): string { @@ -177,11 +185,12 @@ function validateCachedResult(cachedResult: mixed): ?CachedResult { return null; } -class GlobalTransformCache { +class URIBasedGlobalTransformCache { _fetcher: KeyURIFetcher; _fetchResultFromURI: FetchResultFromURI; _profileSet: TransformProfileSet; + _optionsHasher: OptionsHasher; _store: ?KeyResultStore; static FetchFailedError; @@ -194,31 +203,34 @@ class GlobalTransformCache { * of returning the content directly allows for independent and parallel * fetching of each result, that may be arbitrarily large JSON blobs. */ - constructor( - fetchResultURIs: FetchResultURIs, + constructor(props: { fetchResultFromURI: FetchResultFromURI, - storeResults: ?StoreResults, + fetchResultURIs: FetchResultURIs, profiles: Iterable, - ) { - this._fetcher = new KeyURIFetcher(fetchResultURIs); - this._profileSet = new TransformProfileSet(profiles); - this._fetchResultFromURI = fetchResultFromURI; - if (storeResults != null) { - this._store = new KeyResultStore(storeResults); + rootPath: string, + storeResults: StoreResults | null, + }) { + this._fetcher = new KeyURIFetcher(props.fetchResultURIs); + this._profileSet = new TransformProfileSet(props.profiles); + this._fetchResultFromURI = props.fetchResultFromURI; + this._optionsHasher = new OptionsHasher(props.rootPath); + if (props.storeResults != null) { + this._store = new KeyResultStore(props.storeResults); } } /** * Return a key for identifying uniquely a source file. */ - static keyOf(props: FetchProps) { - const stableOptions = globalizeTransformOptions(props.transformOptions); - const digest = crypto.createHash('sha1').update([ - jsonStableStringify(stableOptions), - props.getTransformCacheKey(props.sourceCode, props.filePath, props.transformOptions), - imurmurhash(props.sourceCode).result().toString(), - ].join('$')).digest('hex'); - return `${digest}-${path.basename(props.filePath)}`; + keyOf(props: FetchProps) { + const hash = crypto.createHash('sha1'); + const {sourceCode, filePath, transformOptions} = props; + this._optionsHasher.hashTransformWorkerOptions(hash, transformOptions); + const cacheKey = props.getTransformCacheKey(sourceCode, filePath, transformOptions); + hash.update(JSON.stringify(cacheKey)); + hash.update(crypto.createHash('sha1').update(sourceCode).digest('hex')); + const digest = hash.digest('hex'); + return `${digest}-${path.basename(filePath)}`; } /** @@ -249,8 +261,8 @@ class GlobalTransformCache { * waiting a little time before retring if experience shows it's useful. */ static fetchResultFromURI(uri: string): Promise { - return GlobalTransformCache._fetchResultFromURI(uri).catch(error => { - if (!GlobalTransformCache.shouldRetryAfterThatError(error)) { + return URIBasedGlobalTransformCache._fetchResultFromURI(uri).catch(error => { + if (!URIBasedGlobalTransformCache.shouldRetryAfterThatError(error)) { throw error; } return this._fetchResultFromURI(uri); @@ -284,7 +296,7 @@ class GlobalTransformCache { * key yet, or an error happened, processed separately. */ async fetch(props: FetchProps): Promise { - const uri = await this._fetcher.fetch(GlobalTransformCache.keyOf(props)); + const uri = await this._fetcher.fetch(this.keyOf(props)); if (uri == null) { return null; } @@ -293,12 +305,92 @@ class GlobalTransformCache { store(props: FetchProps, result: CachedResult) { if (this._store != null) { - this._store.store(GlobalTransformCache.keyOf(props), result); + this._store.store(this.keyOf(props), result); } } } -GlobalTransformCache.FetchFailedError = FetchFailedError; +class OptionsHasher { + _rootPath: string; -module.exports = GlobalTransformCache; + constructor(rootPath: string) { + this._rootPath = rootPath; + } + + /** + * This function is extra-conservative with how it hashes the transform + * options. In particular: + * + * * we need to hash paths relative to the root, not the absolute paths, + * otherwise everyone would have a different cache, defeating the + * purpose of global cache; + * * we need to reject any additional field we do not know of, because + * they could contain absolute path, and we absolutely want to process + * these. + * + * Theorically, Flow could help us prevent any other field from being here by + * using *exact* object type. In practice, the transform options are a mix of + * many different fields including the optional Babel fields, and some serious + * cleanup will be necessary to enable rock-solid typing. + */ + hashTransformWorkerOptions(hash: crypto$Hash, options: TransformWorkerOptions): crypto$Hash { + const {dev, minify, platform, transform, extern, ...unknowns} = options; + const unknownKeys = Object.keys(unknowns); + if (unknownKeys.length > 0) { + const message = `these worker option fields are unknown: ${JSON.stringify(unknownKeys)}`; + throw new CannotHashOptionsError(message); + } + // eslint-disable-next-line no-undef, no-bitwise + hash.update(new Buffer([+dev | +minify << 1 | +!!extern << 2])); + hash.update(JSON.stringify(platform)); + return this.hashTransformOptions(hash, transform); + } + + /** + * The transform options contain absolute paths. This can contain, for + * example, the username if someone works their home directory (very likely). + * We get rid of this local data for the global cache, otherwise nobody would + * share the same cache keys. The project roots should not be needed as part + * of the cache key as they should not affect the transformation of a single + * particular file. + */ + hashTransformOptions(hash: crypto$Hash, options: TransformOptions): crypto$Hash { + const {generateSourceMaps, dev, hot, inlineRequires, platform, + preloadedModules, projectRoots, ramGroups, ...unknowns} = options; + const unknownKeys = Object.keys(unknowns); + if (unknownKeys.length > 0) { + const message = `these transform option fields are unknown: ${JSON.stringify(unknownKeys)}`; + throw new CannotHashOptionsError(message); + } + // eslint-disable-next-line no-undef + hash.update(new Buffer([ + // eslint-disable-next-line no-bitwise + +dev | +generateSourceMaps << 1 | +hot << 2 | +!!inlineRequires << 3, + ])); + hash.update(JSON.stringify(platform)); + let relativeBlacklist = []; + if (typeof inlineRequires === 'object') { + relativeBlacklist = this.relativizeFilePaths(Object.keys(inlineRequires.blacklist)); + } + const relativeProjectRoots = this.relativizeFilePaths(projectRoots); + const optionTuple = [relativeBlacklist, preloadedModules, relativeProjectRoots, ramGroups]; + hash.update(JSON.stringify(optionTuple)); + return hash; + } + + relativizeFilePaths(filePaths: Array): Array { + return filePaths.map(filepath => path.relative(this._rootPath, filepath)); + } +} + +class CannotHashOptionsError extends Error { + constructor(message: string) { + super(); + this.message = message; + } +} + +URIBasedGlobalTransformCache.FetchFailedError = FetchFailedError; + +module.exports = {URIBasedGlobalTransformCache, CannotHashOptionsError}; diff --git a/packages/metro-bundler/src/lib/__tests__/GlobalTransformCache-test.js b/packages/metro-bundler/src/lib/__tests__/GlobalTransformCache-test.js index 17fee4a4..94039e8b 100644 --- a/packages/metro-bundler/src/lib/__tests__/GlobalTransformCache-test.js +++ b/packages/metro-bundler/src/lib/__tests__/GlobalTransformCache-test.js @@ -15,8 +15,9 @@ jest.useRealTimers(); const fetchMock = jest.fn(); jest.mock('node-fetch', () => fetchMock); -const GlobalTransformCache = require('../GlobalTransformCache'); +const {URIBasedGlobalTransformCache} = require('../GlobalTransformCache'); const FetchError = require('node-fetch/lib/fetch-error'); +const path = require('path'); async function fetchResultURIs(keys: Array): Promise> { return new Map(keys.map(key => [key, `http://globalcache.com/${key}`])); @@ -33,14 +34,27 @@ async function fetchResultFromURI(uri: string): Promise { describe('GlobalTransformCache', () => { it('fetches results', async () => { - const cache = new GlobalTransformCache(fetchResultURIs, fetchResultFromURI, null, [ - {dev: true, minify: false, platform: 'ios'}, - ]); + const cache = new URIBasedGlobalTransformCache({ + fetchResultFromURI, + fetchResultURIs, + profiles: [{dev: true, minify: false, platform: 'ios'}], + rootPath: __dirname, + storeResults: null, + }); const transformOptions = { dev: true, minify: false, platform: 'ios', - transform: {}, + transform: { + generateSourceMaps: false, + dev: false, + hot: false, + inlineRequires: false, + platform: 'ios', + preloadedModules: [], + projectRoots: [path.join(__dirname, 'root')], + ramGroups: [], + }, }; const result = await Promise.all([cache.fetch({ filePath: 'foo.js', @@ -73,7 +87,8 @@ describe('GlobalTransformCache', () => { it('fetches result', async () => { fetchMock.mockImplementation(defaultFetchMockImpl); - const result = await GlobalTransformCache.fetchResultFromURI('http://globalcache.com/foo'); + const result = await URIBasedGlobalTransformCache + .fetchResultFromURI('http://globalcache.com/foo'); expect(result).toMatchSnapshot(); }); @@ -82,7 +97,8 @@ describe('GlobalTransformCache', () => { fetchMock.mockImplementation(defaultFetchMockImpl); throw new FetchError('timeout!', 'request-timeout'); }); - const result = await GlobalTransformCache.fetchResultFromURI('http://globalcache.com/foo'); + const result = await URIBasedGlobalTransformCache + .fetchResultFromURI('http://globalcache.com/foo'); expect(result).toMatchSnapshot(); }); diff --git a/packages/metro-bundler/src/lib/__tests__/__snapshots__/GlobalTransformCache-test.js.snap b/packages/metro-bundler/src/lib/__tests__/__snapshots__/GlobalTransformCache-test.js.snap index 70472f3a..ed139b38 100644 --- a/packages/metro-bundler/src/lib/__tests__/__snapshots__/GlobalTransformCache-test.js.snap +++ b/packages/metro-bundler/src/lib/__tests__/__snapshots__/GlobalTransformCache-test.js.snap @@ -19,12 +19,12 @@ Object { exports[`GlobalTransformCache fetches results 1`] = ` Array [ Object { - "code": "/* code from http://globalcache.com/17ce731dadd36402f15abb768843cb5e8ecf9ca7-foo.js */", + "code": "/* code from http://globalcache.com/fb94b11256237327e4eef5daa08bad33573b3781-foo.js */", "dependencies": Array [], "dependencyOffsets": Array [], }, Object { - "code": "/* code from http://globalcache.com/d78572916bd877b6ac0e819e99be9fc2954f0e00-bar.js */", + "code": "/* code from http://globalcache.com/b902b4a8841fcea2919c2fe9b21e68d96dff59c9-bar.js */", "dependencies": Array [], "dependencyOffsets": Array [], }, diff --git a/packages/metro-bundler/src/node-haste/Module.js b/packages/metro-bundler/src/node-haste/Module.js index 848822bc..474d2c36 100644 --- a/packages/metro-bundler/src/node-haste/Module.js +++ b/packages/metro-bundler/src/node-haste/Module.js @@ -23,7 +23,7 @@ const jsonStableStringify = require('json-stable-stringify'); const {join: joinPath, relative: relativePath, extname} = require('path'); import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker'; -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; import type {SourceMap} from '../lib/SourceMap'; import type {GetTransformCacheKey} from '../lib/TransformCache'; import type {ReadTransformProps} from '../lib/TransformCache'; diff --git a/packages/metro-bundler/src/node-haste/ModuleCache.js b/packages/metro-bundler/src/node-haste/ModuleCache.js index 316500c5..3622b6c2 100644 --- a/packages/metro-bundler/src/node-haste/ModuleCache.js +++ b/packages/metro-bundler/src/node-haste/ModuleCache.js @@ -16,7 +16,7 @@ const Module = require('./Module'); const Package = require('./Package'); const Polyfill = require('./Polyfill'); -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; import type {GetTransformCacheKey} from '../lib/TransformCache'; import type {Reporter} from '../lib/reporting'; import type Cache from './Cache'; diff --git a/packages/metro-bundler/src/node-haste/index.js b/packages/metro-bundler/src/node-haste/index.js index c0ed382d..6f521fbf 100644 --- a/packages/metro-bundler/src/node-haste/index.js +++ b/packages/metro-bundler/src/node-haste/index.js @@ -38,7 +38,7 @@ const { const {EventEmitter} = require('events'); import type {Options as TransformOptions} from '../JSTransformer/worker/worker'; -import type GlobalTransformCache from '../lib/GlobalTransformCache'; +import type {GlobalTransformCache} from '../lib/GlobalTransformCache'; import type {GetTransformCacheKey} from '../lib/TransformCache'; import type {Reporter} from '../lib/reporting'; import type {ModuleMap} from './DependencyGraph/ResolutionRequest';