packager: GlobalTransformCache: make key globalized

Reviewed By: davidaurelio

Differential Revision: D4835217

fbshipit-source-id: b43456e1e1f83c849a887b07f4f01f8ed0e9df4b

packages/metro-bundler/react-packager.js

@@ -17,7 +17,7 @@ const debug = require('debug');
 const invariant = require('fbjs/lib/invariant');
 
 import type Server from './src/Server';
-import type GlobalTransformCache from './src/lib/GlobalTransformCache';
+import type {GlobalTransformCache} from './src/lib/GlobalTransformCache';
 import type {Reporter} from './src/lib/reporting';
 import type {HasteImpl} from './src/node-haste/Module';
 

packages/metro-bundler/src/Bundler/index.js

@@ -46,13 +46,19 @@ import type {
   TransformOptions,
 } from '../JSTransformer/worker/worker';
 import type {Reporter} from '../lib/reporting';
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
+
+export type ExtraTransformOptions = {
+  +inlineRequires?: {+blacklist: {[string]: true}} | boolean,
+  +preloadedModules?: Array<string> | false,
+  +ramGroups?: Array<string>,
+};
 
 export type GetTransformOptions = (
   mainModuleName: string,
   options: {},
   getDependencies: string => Promise<Array<string>>,
-) => {} | Promise<{}>;
+) => ExtraTransformOptions | Promise<ExtraTransformOptions>;
 
 type Asset = {
   __packager_asset: boolean,

packages/metro-bundler/src/JSTransformer/worker/worker.js

@@ -36,15 +36,19 @@ type Transformer = {
 };
 
 export type TransformOptions = {
+  +dev: boolean,
   generateSourceMaps: boolean,
+  +hot: boolean,
+  +inlineRequires: {+blacklist: {[string]: true}} | boolean,
   platform: string,
-  preloadedModules?: Array<string>,
+  preloadedModules?: Array<string> | false,
   projectRoots: Array<string>,
   ramGroups?: Array<string>,
 } & BabelTransformOptions;
 
 export type Options = {
   +dev: boolean,
+  +extern?: boolean,
   +minify: boolean,
   platform: string,
   transform: TransformOptions,

packages/metro-bundler/src/Resolver/index.js

@@ -24,7 +24,7 @@ import type {Reporter} from '../lib/reporting';
 import type {TransformCode} from '../node-haste/Module';
 import type Cache from '../node-haste/Cache';
 import type {GetTransformCacheKey} from '../lib/TransformCache';
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
 
 type MinifyCode = (filePath: string, code: string, map: SourceMap) =>
   Promise<{code: string, map: SourceMap}>;

packages/metro-bundler/src/Server/index.js

@@ -34,7 +34,7 @@ import type Bundle from '../Bundler/Bundle';
 import type HMRBundle from '../Bundler/HMRBundle';
 import type {Reporter} from '../lib/reporting';
 import type {GetTransformOptions} from '../Bundler';
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
 import type {SourceMap, Symbolicate} from './symbolicate';
 
 const {

packages/metro-bundler/src/lib/GlobalTransformCache.js

@@ -16,22 +16,53 @@ const FetchError = require('node-fetch/lib/fetch-error');
 
 const crypto = require('crypto');
 const fetch = require('node-fetch');
-const imurmurhash = require('imurmurhash');
 const jsonStableStringify = require('json-stable-stringify');
 const path = require('path');
 
-import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
+import type {
+  Options as TransformWorkerOptions,
+  TransformOptions,
+} from '../JSTransformer/worker/worker';
 import type {CachedResult, GetTransformCacheKey} from './TransformCache';
 
+/**
+ * The API that a global transform cache must comply with. To implement a
+ * custom cache, implement this interface and pass it as argument to the
+ * application's top-level `Server` class.
+ */
+export type GlobalTransformCache = {
+  /**
+   * Synchronously determine if it is worth trying to fetch a result from the
+   * cache. This can be used, for instance, to exclude sets of options we know
+   * will never be cached.
+   */
+  shouldFetch(props: FetchProps): boolean,
+
+  /**
+   * Try to fetch a result. It doesn't actually need to fetch from a server,
+   * the global cache could be instantiated locally for example.
+   */
+  fetch(props: FetchProps): Promise<?CachedResult>,
+
+  /**
+   * Try to store a result, without waiting for the success or failure of the
+   * operation. Consequently, the actual storage operation could be done at a
+   * much later point if desired. It is recommended to actually have this
+   * function be a no-op in production, and only do the storage operation from
+   * a script running on your Continuous Integration platform.
+   */
+  store(props: FetchProps, result: CachedResult): void,
+};
+
 type FetchResultURIs = (keys: Array<string>) => Promise<Map<string, string>>;
 type FetchResultFromURI = (uri: string) => Promise<?CachedResult>;
 type StoreResults = (resultsByKey: Map<string, CachedResult>) => Promise<void>;
 
-type FetchProps = {
+export type FetchProps = {
   filePath: string,
   sourceCode: string,
   getTransformCacheKey: GetTransformCacheKey,
-  transformOptions: TransformOptions,
+  transformOptions: TransformWorkerOptions,
 };
 
 type URI = string;
@@ -98,29 +129,6 @@ class KeyResultStore {
 
 }
 
-/**
- * The transform options contain absolute paths. This can contain, for example,
- * the username if someone works their home directory (very likely). We get rid
- * of this local data for the global cache, otherwise nobody would share the
- * same cache keys. The project roots should not be needed as part of the cache
- * key as they should not affect the transformation of a single particular file.
- */
-function globalizeTransformOptions(
-  options: TransformOptions,
-): TransformOptions {
-  const {transform} = options;
-  if (transform == null) {
-    return options;
-  }
-  return {
-    ...options,
-    transform: {
-      ...transform,
-      projectRoots: [],
-    },
-  };
-}
-
 export type TransformProfile = {+dev: boolean, +minify: boolean, +platform: string};
 
 function profileKey({dev, minify, platform}: TransformProfile): string {
@@ -177,11 +185,12 @@ function validateCachedResult(cachedResult: mixed): ?CachedResult {
   return null;
 }
 
-class GlobalTransformCache {
+class URIBasedGlobalTransformCache {
 
   _fetcher: KeyURIFetcher;
   _fetchResultFromURI: FetchResultFromURI;
   _profileSet: TransformProfileSet;
+  _optionsHasher: OptionsHasher;
   _store: ?KeyResultStore;
 
   static FetchFailedError;
@@ -194,31 +203,34 @@ class GlobalTransformCache {
    * of returning the content directly allows for independent and parallel
    * fetching of each result, that may be arbitrarily large JSON blobs.
    */
-  constructor(
+  constructor(props: {
-    fetchResultURIs: FetchResultURIs,
     fetchResultFromURI: FetchResultFromURI,
-    storeResults: ?StoreResults,
+    fetchResultURIs: FetchResultURIs,
     profiles: Iterable<TransformProfile>,
-  ) {
+    rootPath: string,
-    this._fetcher = new KeyURIFetcher(fetchResultURIs);
+    storeResults: StoreResults | null,
-    this._profileSet = new TransformProfileSet(profiles);
+  }) {
-    this._fetchResultFromURI = fetchResultFromURI;
+    this._fetcher = new KeyURIFetcher(props.fetchResultURIs);
-    if (storeResults != null) {
+    this._profileSet = new TransformProfileSet(props.profiles);
-      this._store = new KeyResultStore(storeResults);
+    this._fetchResultFromURI = props.fetchResultFromURI;
+    this._optionsHasher = new OptionsHasher(props.rootPath);
+    if (props.storeResults != null) {
+      this._store = new KeyResultStore(props.storeResults);
     }
   }
 
   /**
    * Return a key for identifying uniquely a source file.
    */
-  static keyOf(props: FetchProps) {
+  keyOf(props: FetchProps) {
-    const stableOptions = globalizeTransformOptions(props.transformOptions);
+    const hash = crypto.createHash('sha1');
-    const digest = crypto.createHash('sha1').update([
+    const {sourceCode, filePath, transformOptions} = props;
-      jsonStableStringify(stableOptions),
+    this._optionsHasher.hashTransformWorkerOptions(hash, transformOptions);
-      props.getTransformCacheKey(props.sourceCode, props.filePath, props.transformOptions),
+    const cacheKey = props.getTransformCacheKey(sourceCode, filePath, transformOptions);
-      imurmurhash(props.sourceCode).result().toString(),
+    hash.update(JSON.stringify(cacheKey));
-    ].join('$')).digest('hex');
+    hash.update(crypto.createHash('sha1').update(sourceCode).digest('hex'));
-    return `${digest}-${path.basename(props.filePath)}`;
+    const digest = hash.digest('hex');
+    return `${digest}-${path.basename(filePath)}`;
   }
 
   /**
@@ -249,8 +261,8 @@ class GlobalTransformCache {
    * waiting a little time before retring if experience shows it's useful.
    */
   static fetchResultFromURI(uri: string): Promise<CachedResult> {
-    return GlobalTransformCache._fetchResultFromURI(uri).catch(error => {
+    return URIBasedGlobalTransformCache._fetchResultFromURI(uri).catch(error => {
-      if (!GlobalTransformCache.shouldRetryAfterThatError(error)) {
+      if (!URIBasedGlobalTransformCache.shouldRetryAfterThatError(error)) {
         throw error;
       }
       return this._fetchResultFromURI(uri);
@@ -284,7 +296,7 @@ class GlobalTransformCache {
    * key yet, or an error happened, processed separately.
    */
   async fetch(props: FetchProps): Promise<?CachedResult> {
-    const uri = await this._fetcher.fetch(GlobalTransformCache.keyOf(props));
+    const uri = await this._fetcher.fetch(this.keyOf(props));
     if (uri == null) {
       return null;
     }
@@ -293,12 +305,92 @@ class GlobalTransformCache {
 
   store(props: FetchProps, result: CachedResult) {
     if (this._store != null) {
-      this._store.store(GlobalTransformCache.keyOf(props), result);
+      this._store.store(this.keyOf(props), result);
     }
   }
 
 }
 
-GlobalTransformCache.FetchFailedError = FetchFailedError;
+class OptionsHasher {
+  _rootPath: string;
 
-module.exports = GlobalTransformCache;
+  constructor(rootPath: string) {
+    this._rootPath = rootPath;
+  }
+
+  /**
+   * This function is extra-conservative with how it hashes the transform
+   * options. In particular:
+   *
+   *     * we need to hash paths relative to the root, not the absolute paths,
+   *       otherwise everyone would have a different cache, defeating the
+   *       purpose of global cache;
+   *     * we need to reject any additional field we do not know of, because
+   *       they could contain absolute path, and we absolutely want to process
+   *       these.
+   *
+   * Theorically, Flow could help us prevent any other field from being here by
+   * using *exact* object type. In practice, the transform options are a mix of
+   * many different fields including the optional Babel fields, and some serious
+   * cleanup will be necessary to enable rock-solid typing.
+   */
+  hashTransformWorkerOptions(hash: crypto$Hash, options: TransformWorkerOptions): crypto$Hash {
+    const {dev, minify, platform, transform, extern, ...unknowns} = options;
+    const unknownKeys = Object.keys(unknowns);
+    if (unknownKeys.length > 0) {
+      const message = `these worker option fields are unknown: ${JSON.stringify(unknownKeys)}`;
+      throw new CannotHashOptionsError(message);
+    }
+    // eslint-disable-next-line no-undef, no-bitwise
+    hash.update(new Buffer([+dev | +minify << 1 | +!!extern << 2]));
+    hash.update(JSON.stringify(platform));
+    return this.hashTransformOptions(hash, transform);
+  }
+
+  /**
+   * The transform options contain absolute paths. This can contain, for
+   * example, the username if someone works their home directory (very likely).
+   * We get rid of this local data for the global cache, otherwise nobody would
+   * share the same cache keys. The project roots should not be needed as part
+   * of the cache key as they should not affect the transformation of a single
+   * particular file.
+   */
+  hashTransformOptions(hash: crypto$Hash, options: TransformOptions): crypto$Hash {
+    const {generateSourceMaps, dev, hot, inlineRequires, platform,
+      preloadedModules, projectRoots, ramGroups, ...unknowns} = options;
+    const unknownKeys = Object.keys(unknowns);
+    if (unknownKeys.length > 0) {
+      const message = `these transform option fields are unknown: ${JSON.stringify(unknownKeys)}`;
+      throw new CannotHashOptionsError(message);
+    }
+    // eslint-disable-next-line no-undef
+    hash.update(new Buffer([
+      // eslint-disable-next-line no-bitwise
+      +dev | +generateSourceMaps << 1 | +hot << 2 | +!!inlineRequires << 3,
+    ]));
+    hash.update(JSON.stringify(platform));
+    let relativeBlacklist = [];
+    if (typeof inlineRequires === 'object') {
+      relativeBlacklist = this.relativizeFilePaths(Object.keys(inlineRequires.blacklist));
+    }
+    const relativeProjectRoots = this.relativizeFilePaths(projectRoots);
+    const optionTuple = [relativeBlacklist, preloadedModules, relativeProjectRoots, ramGroups];
+    hash.update(JSON.stringify(optionTuple));
+    return hash;
+  }
+
+  relativizeFilePaths(filePaths: Array<string>): Array<string> {
+    return filePaths.map(filepath => path.relative(this._rootPath, filepath));
+  }
+}
+
+class CannotHashOptionsError extends Error {
+  constructor(message: string) {
+    super();
+    this.message = message;
+  }
+}
+
+URIBasedGlobalTransformCache.FetchFailedError = FetchFailedError;
+
+module.exports = {URIBasedGlobalTransformCache, CannotHashOptionsError};

packages/metro-bundler/src/lib/__tests__/GlobalTransformCache-test.js

@@ -15,8 +15,9 @@ jest.useRealTimers();
 const fetchMock = jest.fn();
 jest.mock('node-fetch', () => fetchMock);
 
-const GlobalTransformCache = require('../GlobalTransformCache');
+const {URIBasedGlobalTransformCache} = require('../GlobalTransformCache');
 const FetchError = require('node-fetch/lib/fetch-error');
+const path = require('path');
 
 async function fetchResultURIs(keys: Array<string>): Promise<Map<string, string>> {
   return new Map(keys.map(key => [key, `http://globalcache.com/${key}`]));
@@ -33,14 +34,27 @@ async function fetchResultFromURI(uri: string): Promise<?CachedResult> {
 describe('GlobalTransformCache', () => {
 
   it('fetches results', async () => {
-    const cache = new GlobalTransformCache(fetchResultURIs, fetchResultFromURI, null, [
+    const cache = new URIBasedGlobalTransformCache({
-      {dev: true, minify: false, platform: 'ios'},
+      fetchResultFromURI,
-    ]);
+      fetchResultURIs,
+      profiles: [{dev: true, minify: false, platform: 'ios'}],
+      rootPath: __dirname,
+      storeResults: null,
+    });
     const transformOptions = {
       dev: true,
       minify: false,
       platform: 'ios',
-      transform: {},
+      transform: {
+        generateSourceMaps: false,
+        dev: false,
+        hot: false,
+        inlineRequires: false,
+        platform: 'ios',
+        preloadedModules: [],
+        projectRoots: [path.join(__dirname, 'root')],
+        ramGroups: [],
+      },
     };
     const result = await Promise.all([cache.fetch({
       filePath: 'foo.js',
@@ -73,7 +87,8 @@ describe('GlobalTransformCache', () => {
 
     it('fetches result', async () => {
       fetchMock.mockImplementation(defaultFetchMockImpl);
-      const result = await GlobalTransformCache.fetchResultFromURI('http://globalcache.com/foo');
+      const result = await URIBasedGlobalTransformCache
+        .fetchResultFromURI('http://globalcache.com/foo');
       expect(result).toMatchSnapshot();
     });
 
@@ -82,7 +97,8 @@ describe('GlobalTransformCache', () => {
         fetchMock.mockImplementation(defaultFetchMockImpl);
         throw new FetchError('timeout!', 'request-timeout');
       });
-      const result = await GlobalTransformCache.fetchResultFromURI('http://globalcache.com/foo');
+      const result = await URIBasedGlobalTransformCache
+        .fetchResultFromURI('http://globalcache.com/foo');
       expect(result).toMatchSnapshot();
     });
 

packages/metro-bundler/src/lib/__tests__/__snapshots__/GlobalTransformCache-test.js.snap

@@ -19,12 +19,12 @@ Object {
 exports[`GlobalTransformCache fetches results 1`] = `
 Array [
   Object {
-    "code": "/* code from http://globalcache.com/17ce731dadd36402f15abb768843cb5e8ecf9ca7-foo.js */",
+    "code": "/* code from http://globalcache.com/fb94b11256237327e4eef5daa08bad33573b3781-foo.js */",
     "dependencies": Array [],
     "dependencyOffsets": Array [],
   },
   Object {
-    "code": "/* code from http://globalcache.com/d78572916bd877b6ac0e819e99be9fc2954f0e00-bar.js */",
+    "code": "/* code from http://globalcache.com/b902b4a8841fcea2919c2fe9b21e68d96dff59c9-bar.js */",
     "dependencies": Array [],
     "dependencyOffsets": Array [],
   },

packages/metro-bundler/src/node-haste/Module.js

@@ -23,7 +23,7 @@ const jsonStableStringify = require('json-stable-stringify');
 const {join: joinPath, relative: relativePath, extname} = require('path');
 
 import type {TransformedCode, Options as TransformOptions} from '../JSTransformer/worker/worker';
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
 import type {SourceMap} from '../lib/SourceMap';
 import type {GetTransformCacheKey} from '../lib/TransformCache';
 import type {ReadTransformProps} from '../lib/TransformCache';

packages/metro-bundler/src/node-haste/ModuleCache.js

@@ -16,7 +16,7 @@ const Module = require('./Module');
 const Package = require('./Package');
 const Polyfill = require('./Polyfill');
 
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
 import type {GetTransformCacheKey} from '../lib/TransformCache';
 import type {Reporter} from '../lib/reporting';
 import type Cache from './Cache';

packages/metro-bundler/src/node-haste/index.js

@@ -38,7 +38,7 @@ const {
 const {EventEmitter} = require('events');
 
 import type {Options as TransformOptions} from '../JSTransformer/worker/worker';
-import type GlobalTransformCache from '../lib/GlobalTransformCache';
+import type {GlobalTransformCache} from '../lib/GlobalTransformCache';
 import type {GetTransformCacheKey} from '../lib/TransformCache';
 import type {Reporter} from '../lib/reporting';
 import type {ModuleMap} from './DependencyGraph/ResolutionRequest';