packager: add GlobalTransformCache

Reviewed By: davidaurelio

Differential Revision: D4175938

fbshipit-source-id: 1f57d594b4c8c8189feb2ea6d4d4011870ffd85f

react-packager/src/Server/__tests__/Server-test.js

@@ -19,7 +19,8 @@ jest.setMock('worker-farm', function() { return () => {}; })
     .mock('../../AssetServer')
     .mock('../../lib/declareOpts')
     .mock('../../node-haste')
-    .mock('../../Logger');
+    .mock('../../Logger')
+    .mock('../../lib/GlobalTransformCache');
 
 describe('processRequest', () => {
   let SourceMapConsumer, Bundler, Server, AssetServer, Promise;

react-packager/src/lib/GlobalTransformCache.js

@@ -0,0 +1,198 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+const debounce = require('lodash/debounce');
+const imurmurhash = require('imurmurhash');
+const jsonStableStringify = require('json-stable-stringify');
+const path = require('path');
+const request = require('request');
+const toFixedHex = require('./toFixedHex');
+
+import type {CachedResult} from './TransformCache';
+
+const SINGLE_REQUEST_MAX_KEYS = 100;
+const AGGREGATION_DELAY_MS = 100;
+
+type FetchResultURIs = (
+  keys: Array<string>,
+  callback: (error?: Error, results?: Map<string, string>) => void,
+) => mixed;
+
+type FetchProps = {
+  filePath: string,
+  sourceCode: string,
+  transformCacheKey: string,
+  transformOptions: mixed,
+};
+
+type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
+type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
+
+/**
+ * We aggregate the requests to do a single request for many keys. It also
+ * ensures we do a single request at a time to avoid pressuring the I/O.
+ */
+class KeyURIFetcher {
+
+  _fetchResultURIs: FetchResultURIs;
+  _pendingQueries: Array<{key: string, callback: FetchURICallback}>;
+  _isProcessing: boolean;
+  _processQueriesDebounced: () => void;
+  _processQueries: () => void;
+
+  /**
+   * Fetch the pending keys right now, if any and if we're not already doing
+   * so in parallel. At the end of the fetch, we trigger a new batch fetching
+   * recursively.
+   */
+  _processQueries() {
+    const {_pendingQueries} = this;
+    if (_pendingQueries.length === 0 || this._isProcessing) {
+      return;
+    }
+    this._isProcessing = true;
+    const queries = _pendingQueries.splice(0, SINGLE_REQUEST_MAX_KEYS);
+    const keys = queries.map(query => query.key);
+    this._fetchResultURIs(keys, (error, results) => {
+      queries.forEach(query => {
+        query.callback(error, results && results.get(query.key));
+      });
+      this._isProcessing = false;
+      process.nextTick(this._processQueries);
+    });
+  }
+
+  /**
+   * Enqueue the fetching of a particular key.
+   */
+  fetch(key: string, callback: FetchURICallback) {
+    this._pendingQueries.push({key, callback});
+    this._processQueriesDebounced();
+  }
+
+  constructor(fetchResultURIs: FetchResultURIs) {
+    this._fetchResultURIs = fetchResultURIs;
+    this._pendingQueries = [];
+    this._isProcessing = false;
+    this._processQueries = this._processQueries.bind(this);
+    this._processQueriesDebounced =
+      debounce(this._processQueries, AGGREGATION_DELAY_MS);
+  }
+
+}
+
+function validateCachedResult(cachedResult: mixed): ?CachedResult {
+  if (
+    cachedResult != null &&
+    typeof cachedResult === 'object' &&
+    typeof cachedResult.code === 'string' &&
+    Array.isArray(cachedResult.dependencies) &&
+    cachedResult.dependencies.every(dep => typeof dep === 'string') &&
+    Array.isArray(cachedResult.dependencyOffsets) &&
+    cachedResult.dependencyOffsets.every(offset => typeof offset === 'number')
+  ) {
+    return (cachedResult: any);
+  }
+  return undefined;
+}
+
+/**
+ * One can enable the global cache by calling configure() from a custom CLI
+ * script. Eventually we may make it more flexible.
+ */
+class GlobalTransformCache {
+
+  _fetcher: KeyURIFetcher;
+  static _global: ?GlobalTransformCache;
+
+  constructor(fetchResultURIs: FetchResultURIs) {
+    this._fetcher = new KeyURIFetcher(fetchResultURIs);
+  }
+
+  /**
+   * Return a key for identifying uniquely a source file.
+   */
+  static keyOf(props: FetchProps) {
+    const sourceDigest = toFixedHex(8, imurmurhash(props.sourceCode).result());
+    const optionsHash = imurmurhash()
+      .hash(jsonStableStringify(props.transformOptions) || '')
+      .hash(props.transformCacheKey)
+      .result();
+    const optionsDigest = toFixedHex(8, optionsHash);
+    return (
+      `${optionsDigest}${sourceDigest}` +
+      `${path.basename(props.filePath)}`
+    );
+  }
+
+  /**
+   * We may want to improve that logic to return a stream instead of the whole
+   * blob of transformed results. However the results are generally only a few
+   * megabytes each.
+   */
+  _fetchFromURI(uri: string, callback: FetchCallback) {
+    request.get({uri, json: true}, (error, response, unvalidatedResult) => {
+      if (error != null) {
+        callback(error);
+        return;
+      }
+      if (response.statusCode !== 200) {
+        callback(new Error(
+          `Unexpected HTTP status code: ${response.statusCode}`,
+        ));
+        return;
+      }
+      const result = validateCachedResult(unvalidatedResult);
+      if (result == null) {
+        callback(new Error('Invalid result returned by server.'));
+        return;
+      }
+      callback(undefined, result);
+    });
+  }
+
+  fetch(props: FetchProps, callback: FetchCallback) {
+    this._fetcher.fetch(GlobalTransformCache.keyOf(props), (error, uri) => {
+      if (error != null) {
+        callback(error);
+      } else {
+        if (uri == null) {
+          callback();
+          return;
+        }
+        this._fetchFromURI(uri, callback);
+      }
+    });
+  }
+
+  /**
+   * For using the global cache one needs to have some kind of central key-value
+   * store that gets prefilled using keyOf() and the transformed results. The
+   * fetching function should provide a mapping of keys to URIs. The files
+   * referred by these URIs contains the transform results. Using URIs instead
+   * of returning the content directly allows for independent fetching of each
+   * result.
+   */
+  static configure(fetchResultURIs: FetchResultURIs) {
+    GlobalTransformCache._global = new GlobalTransformCache(fetchResultURIs);
+  }
+
+  static get() {
+    return GlobalTransformCache._global;
+  }
+
+}
+
+GlobalTransformCache._global = null;
+
+module.exports = GlobalTransformCache;

react-packager/src/lib/TransformCache.js

@@ -22,6 +22,7 @@ const jsonStableStringify = require('json-stable-stringify');
 const mkdirp = require('mkdirp');
 const path = require('path');
 const rimraf = require('rimraf');
+const toFixedHex = require('./toFixedHex');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 
 const CACHE_NAME = 'react-native-packager-cache';
@@ -66,15 +67,14 @@ function getCacheFilePaths(props: {
   const hasher = imurmurhash()
     .hash(props.filePath)
     .hash(jsonStableStringify(props.transformOptions) || '');
-  let hash = hasher.result().toString(16);
-  hash = Array(8 - hash.length + 1).join('0') + hash;
+  const hash = toFixedHex(8, hasher.result());
   const prefix = hash.substr(0, 2);
   const fileName = `${hash.substr(2)}${path.basename(props.filePath)}`;
   const base = path.join(getCacheDirPath(), prefix, fileName);
   return {transformedCode: base, metadata: base + '.meta'};
 }
 
-type CachedResult = {
+export type CachedResult = {
   code: string,
   dependencies: Array<string>,
   dependencyOffsets: Array<number>,
@@ -135,7 +135,7 @@ function writeSync(props: {
   ]));
 }
 
-type CacheOptions = {resetCache?: boolean};
+export type CacheOptions = {resetCache?: boolean};
 
 /* 1 day */
 const GARBAGE_COLLECTION_PERIOD = 24 * 60 * 60 * 1000;
@@ -272,6 +272,14 @@ function readMetadataFileSync(
   };
 }
 
+export type ReadTransformProps = {
+  filePath: string,
+  sourceCode: string,
+  transformOptions: mixed,
+  transformCacheKey: string,
+  cacheOptions: CacheOptions,
+};
+
 /**
  * We verify the source hash matches to ensure we always favor rebuilding when
  * source change (rather than just using fs.mtime(), a bit less robust).
@@ -285,13 +293,7 @@ function readMetadataFileSync(
  * Meanwhile we store transforms with different options in different files so
  * that it is fast to switch between ex. minified, or not.
  */
-function readSync(props: {
-  filePath: string,
-  sourceCode: string,
-  transformOptions: mixed,
-  transformCacheKey: string,
-  cacheOptions: CacheOptions,
-}): ?CachedResult {
+function readSync(props: ReadTransformProps): ?CachedResult {
   GARBAGE_COLLECTOR.collectIfNecessarySync(props.cacheOptions);
   const cacheFilePaths = getCacheFilePaths(props);
   let metadata, transformedCode;

react-packager/src/lib/__mocks__/GlobalTransformCache.js

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+function get() {
+  return null;
+}
+
+module.exports = {get};

react-packager/src/lib/__tests__/TransformCache-test.js

@@ -12,7 +12,9 @@
 jest
   .dontMock('imurmurhash')
   .dontMock('json-stable-stringify')
-  .dontMock('../TransformCache');
+  .dontMock('../TransformCache')
+  .dontMock('../toFixedHex')
+  .dontMock('left-pad');
 
 const imurmurhash = require('imurmurhash');
 

react-packager/src/lib/toFixedHex.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+const leftPad = require('left-pad');
+
+function toFixedHex(length: number, number: number): string {
+  return leftPad(number.toString(16), length, '0');
+}
+
+module.exports = toFixedHex;

react-packager/src/node-haste/Module.js

@@ -11,8 +11,10 @@
 
 'use strict';
 
+const GlobalTransformCache = require('../lib/GlobalTransformCache');
 const TransformCache = require('../lib/TransformCache');
 
+const chalk = require('chalk');
 const crypto = require('crypto');
 const docblock = require('./DependencyGraph/docblock');
 const invariant = require('invariant');
@@ -22,6 +24,7 @@ const jsonStableStringify = require('json-stable-stringify');
 const {join: joinPath, relative: relativePath, extname} = require('path');
 
 import type {TransformedCode} from '../JSTransformer/worker/worker';
+import type {ReadTransformProps} from '../lib/TransformCache';
 import type Cache from './Cache';
 import type DependencyGraphHelpers from './DependencyGraph/DependencyGraphHelpers';
 import type ModuleCache from './ModuleCache';
@@ -73,6 +76,8 @@ class Module {
   _readSourceCodePromise: Promise<string>;
   _readPromises: Map<string, Promise<ReadResult>>;
 
+  static _useGlobalCache: boolean;
+
   constructor({
     file,
     fastfs,
@@ -214,33 +219,63 @@ class Module {
     return {...result, id, source};
   }
 
-  _transformAndCache(
-    transformOptions: mixed,
+  _transformCodeForCallback(
+    cacheProps: ReadTransformProps,
     callback: (error: ?Error, result: ?TransformedCode) => void,
   ) {
-    const {_transformCode, _transformCacheKey} = this;
+    const {_transformCode} = this;
     invariant(_transformCode != null, 'missing code transform funtion');
-    invariant(_transformCacheKey != null, 'missing cache key');
-    this._readSourceCode()
-      .then(sourceCode =>
-        _transformCode(this, sourceCode, transformOptions)
-          .then(freshResult => {
-            TransformCache.writeSync({
-              filePath: this.path,
-              sourceCode,
-              transformCacheKey: _transformCacheKey,
-              transformOptions,
-              result: freshResult,
-            });
-            return freshResult;
-          })
-      )
-      .then(
-        freshResult => process.nextTick(callback, null, freshResult),
+    const {sourceCode, transformOptions} = cacheProps;
+    return _transformCode(this, sourceCode, transformOptions).then(
+      freshResult => process.nextTick(callback, undefined, freshResult),
       error => process.nextTick(callback, error),
     );
   }
 
+  _getTransformedCode(
+    cacheProps: ReadTransformProps,
+    callback: (error: ?Error, result: ?TransformedCode) => void,
+  ) {
+    const globalCache = GlobalTransformCache.get();
+    if (!Module._useGlobalCache || globalCache == null) {
+      this._transformCodeForCallback(cacheProps, callback);
+      return;
+    }
+    globalCache.fetch(cacheProps, (globalCacheError, globalCachedResult) => {
+      if (globalCacheError != null && Module._useGlobalCache) {
+        console.log(chalk.red(
+          '\nWarning: the global cache failed with error:',
+        ));
+        console.log(chalk.red(globalCacheError.stack));
+        console.log(chalk.red(
+          'The global cache will be DISABLED for the ' +
+            'remainder of the transformation.',
+        ));
+        Module._useGlobalCache = false;
+      }
+      if (globalCacheError != null || globalCachedResult == null) {
+        this._transformCodeForCallback(cacheProps, callback);
+        return;
+      }
+      callback(undefined, globalCachedResult);
+    });
+  }
+
+  _getAndCacheTransformedCode(
+    cacheProps: ReadTransformProps,
+    callback: (error: ?Error, result: ?TransformedCode) => void,
+  ) {
+    this._getTransformedCode(cacheProps, (error, result) => {
+      if (error) {
+        callback(error);
+        return;
+      }
+      invariant(result != null, 'missing result');
+      TransformCache.writeSync({...cacheProps, result});
+      callback(undefined, result);
+    });
+  }
+
   /**
    * Read everything about a module: source code, transformed code,
    * dependencies, etc. The overall process is to read the cache first, and if
@@ -264,20 +299,20 @@ class Module {
       }
       const transformCacheKey = this._transformCacheKey;
       invariant(transformCacheKey != null, 'missing transform cache key');
-      const cachedResult =
-        TransformCache.readSync({
+      const cacheProps = {
         filePath: this.path,
         sourceCode,
         transformCacheKey,
         transformOptions,
         cacheOptions: this._options,
-        });
+      };
+      const cachedResult = TransformCache.readSync(cacheProps);
       if (cachedResult) {
-        return this._finalizeReadResult(sourceCode, id, extern, cachedResult);
+        return Promise.resolve(this._finalizeReadResult(sourceCode, id, extern, cachedResult));
       }
       return new Promise((resolve, reject) => {
-        this._transformAndCache(
-          transformOptions,
+        this._getAndCacheTransformedCode(
+          cacheProps,
           (transformError, freshResult) => {
             if (transformError) {
               reject(transformError);
@@ -320,6 +355,8 @@ class Module {
   }
 }
 
+Module._useGlobalCache = true;
+
 // use weak map to speed up hash creation of known objects
 const knownHashes = new WeakMap();
 function stableObjectHash(object) {