Add HTTP store

Reviewed By: jeanlauliac

Differential Revision: D7382414

fbshipit-source-id: 2c6cdb9330137718f49771bafdabe2b6d24e7ba8

packages/metro-cache/src/index.js

@@ -12,6 +12,7 @@
 
 const Cache = require('./Cache');
 const FileStore = require('./stores/FileStore');
+const HttpStore = require('./stores/HttpStore');
 const PersistedMapStore = require('./stores/PersistedMapStore');
 
 const stableHash = require('./stableHash');
@@ -20,6 +21,7 @@ export type {CacheStore} from './types.flow';
 
 module.exports.Cache = Cache;
 module.exports.FileStore = FileStore;
+module.exports.HttpStore = HttpStore;
 module.exports.PersistedMapStore = PersistedMapStore;
 
 module.exports.stableHash = stableHash;

packages/metro-cache/src/stores/HttpStore.js

@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) 2018-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @flow
+ */
+
+'use strict';
+
+const http = require('http');
+const https = require('https');
+const url = require('url');
+const zlib = require('zlib');
+
+import type {TransformedCode} from 'metro/src/JSTransformer/worker';
+
+export type Options = {|
+  endpoint: string,
+  timeout?: number,
+|};
+
+const ZLIB_OPTIONS = {
+  level: 9,
+};
+
+class HttpStore {
+  _module: typeof http | typeof https;
+  _timeout: number;
+
+  _host: string;
+  _port: number;
+  _path: string;
+
+  _getAgent: http$Agent;
+  _setAgent: http$Agent;
+
+  constructor(options: Options) {
+    const uri = url.parse(options.endpoint);
+    const module = uri.protocol === 'http:' ? http : https;
+
+    const agentConfig = {
+      keepAlive: true,
+      keepAliveMsecs: options.timeout || 5000,
+      maxSockets: 64,
+      maxFreeSockets: 64,
+    };
+
+    if (!uri.hostname || !uri.pathname) {
+      throw new TypeError('Invalid endpoint: ' + options.endpoint);
+    }
+
+    this._module = module;
+    this._timeout = options.timeout || 5000;
+
+    this._host = uri.hostname;
+    this._path = uri.pathname;
+    this._port = +uri.port;
+
+    this._getAgent = new module.Agent(agentConfig);
+    this._setAgent = new module.Agent(agentConfig);
+  }
+
+  get(key: Buffer): Promise<?TransformedCode> {
+    return new Promise((resolve, reject) => {
+      const options = {
+        agent: this._getAgent,
+        host: this._host,
+        method: 'GET',
+        path: this._path + '/' + key.toString('hex'),
+        port: this._port,
+        timeout: this._timeout,
+      };
+
+      const req = this._module.request(options, res => {
+        let data = '';
+
+        if (res.statusCode === 404) {
+          resolve(null);
+          return;
+        } else if (res.statusCode !== 200) {
+          reject(new Error('HTTP error: ' + res.statusCode));
+          return;
+        }
+
+        const gunzipped = res.pipe(zlib.createGunzip());
+
+        gunzipped.on('data', chunk => {
+          data += chunk.toString();
+        });
+
+        gunzipped.on('error', err => {
+          reject(err);
+        });
+
+        gunzipped.on('end', () => {
+          try {
+            resolve(JSON.parse(data));
+          } catch (err) {
+            reject(err);
+          }
+        });
+
+        res.on('error', err => gunzipped.emit('error', err));
+      });
+
+      req.end();
+    });
+  }
+
+  set(key: Buffer, value: TransformedCode): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const gzip = zlib.createGzip(ZLIB_OPTIONS);
+
+      const options = {
+        agent: this._setAgent,
+        host: this._host,
+        method: 'PUT',
+        path: this._path + '/' + key.toString('hex'),
+        port: this._port,
+        timeout: this._timeout,
+      };
+
+      const req = this._module.request(options, res => {
+        res.on('error', err => {
+          reject(err);
+        });
+
+        res.on('data', () => {
+          // Do nothing. It is needed so node thinks we are consuming responses.
+        });
+
+        res.on('end', () => {
+          resolve();
+        });
+      });
+
+      gzip.pipe(req);
+      gzip.end(JSON.stringify(value) || 'null');
+    });
+  }
+}
+
+module.exports = HttpStore;

packages/metro-cache/src/stores/__tests__/HttpStore-test.js

@@ -0,0 +1,202 @@
+/**
+ * Copyright (c) 2018-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+javascript_foundation
+ * @format
+ */
+
+'use strict';
+
+const zlib = require('zlib');
+
+const {PassThrough} = require('stream');
+
+describe('HttpStore', () => {
+  let HttpStore;
+  let httpPassThrough;
+
+  function responseHttpOk(data) {
+    const res = Object.assign(new PassThrough(), {
+      statusCode: 200,
+    });
+
+    process.nextTick(() => {
+      res.write(zlib.gzipSync(data));
+      res.end();
+    });
+
+    return res;
+  }
+
+  function responseHttpError(code) {
+    return Object.assign(new PassThrough(), {
+      statusCode: code,
+    });
+  }
+
+  function responseError(err) {
+    const res = Object.assign(new PassThrough(), {
+      statusCode: 200,
+    });
+
+    process.nextTick(() => {
+      res.emit('error', err);
+    });
+
+    return res;
+  }
+
+  beforeEach(() => {
+    jest
+      .resetModules()
+      .resetAllMocks()
+      .useFakeTimers()
+      .mock('http')
+      .mock('https');
+
+    httpPassThrough = new PassThrough();
+    require('http').request.mockReturnValue(httpPassThrough);
+    require('https').request.mockReturnValue(httpPassThrough);
+
+    HttpStore = require('../HttpStore');
+  });
+
+  it('works with HTTP and HTTPS', () => {
+    const httpStore = new HttpStore({endpoint: 'http://example.com'});
+    const httpsStore = new HttpStore({endpoint: 'https://example.com'});
+
+    httpStore.get(Buffer.from('foo'));
+    expect(require('http').request).toHaveBeenCalledTimes(1);
+    expect(require('https').request).not.toHaveBeenCalled();
+
+    jest.resetAllMocks();
+
+    httpsStore.get(Buffer.from('foo'));
+    expect(require('http').request).not.toHaveBeenCalled();
+    expect(require('https').request).toHaveBeenCalledTimes(1);
+  });
+
+  it('gets using the network via GET method', async () => {
+    const store = new HttpStore({endpoint: 'http://www.example.com/endpoint'});
+    const promise = store.get(Buffer.from('key'));
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('GET');
+    expect(opts.host).toEqual('www.example.com');
+    expect(opts.path).toEqual('/endpoint/6b6579');
+    expect(opts.timeout).toEqual(5000);
+
+    callback(responseHttpOk(JSON.stringify({foo: 42})));
+    jest.runAllTimers();
+
+    expect(await promise).toEqual({foo: 42});
+  });
+
+  it('resolves with "null" when HTTP 404 is returned', async () => {
+    const store = new HttpStore({endpoint: 'http://example.com'});
+    const promise = store.get(Buffer.from('key'));
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('GET');
+
+    callback(responseHttpError(404));
+    jest.runAllTimers();
+
+    expect(await promise).toEqual(null);
+  });
+
+  it('rejects when an HTTP different of 200/404 is returned', done => {
+    const store = new HttpStore({endpoint: 'http://example.com'});
+    const promise = store.get(Buffer.from('key'));
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('GET');
+
+    callback(responseHttpError(503)); // Intentionally unterminated JSON.
+    jest.runAllTimers();
+
+    promise.catch(err => {
+      expect(err).toBeInstanceOf(Error);
+      expect(err.message).toMatch(/HTTP error: 503/);
+      done();
+    });
+  });
+
+  it('rejects when it gets an invalid JSON response', done => {
+    const store = new HttpStore({endpoint: 'http://example.com'});
+    const promise = store.get(Buffer.from('key'));
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('GET');
+
+    callback(responseHttpOk('{"foo": 4')); // Intentionally unterminated JSON.
+    jest.runAllTimers();
+
+    promise.catch(err => {
+      expect(err).toBeInstanceOf(SyntaxError);
+      done();
+    });
+  });
+
+  it('rejects when the HTTP layer throws', done => {
+    const store = new HttpStore({endpoint: 'http://example.com'});
+    const promise = store.get(Buffer.from('key'));
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('GET');
+
+    callback(responseError(new Error('ENOTFOUND')));
+    jest.runAllTimers();
+
+    promise.catch(err => {
+      expect(err).toBeInstanceOf(Error);
+      expect(err.message).toBe('ENOTFOUND');
+      done();
+    });
+  });
+
+  it('sets using the network via PUT method', done => {
+    const store = new HttpStore({endpoint: 'http://www.example.com/endpoint'});
+    const promise = store.set(Buffer.from('key-set'), {foo: 42});
+    const [opts, callback] = require('http').request.mock.calls[0];
+    const buf = [];
+
+    expect(opts.method).toEqual('PUT');
+    expect(opts.host).toEqual('www.example.com');
+    expect(opts.path).toEqual('/endpoint/6b65792d736574');
+    expect(opts.timeout).toEqual(5000);
+
+    callback(responseHttpOk(''));
+
+    httpPassThrough.on('data', chunk => {
+      buf.push(chunk);
+    });
+
+    httpPassThrough.on('end', async () => {
+      expect(zlib.gunzipSync(Buffer.concat(buf)).toString()).toBe('{"foo":42}');
+      await promise; // Ensure that the setting promise successfully finishes.
+
+      done();
+    });
+  });
+
+  it('rejects when setting and HTTP fails', done => {
+    const store = new HttpStore({endpoint: 'http://example.com'});
+    const promise = store.set(Buffer.from('key-set'), {foo: 42});
+    const [opts, callback] = require('http').request.mock.calls[0];
+
+    expect(opts.method).toEqual('PUT');
+
+    callback(responseError(new Error('ENOTFOUND')));
+    jest.runAllTimers();
+
+    promise.catch(err => {
+      expect(err).toBeInstanceOf(Error);
+      expect(err.message).toBe('ENOTFOUND');
+      done();
+    });
+  });
+});

packages/metro/src/Bundler.js

@@ -177,6 +177,7 @@ class Bundler {
       opts.assetExts,
       opts.assetRegistryPath,
       getTransformCacheKey,
+      'experimental',
     ]).toString('binary');
 
     this._projectRoots = opts.projectRoots;

packages/metro/src/JSTransformer/worker.js

@@ -127,10 +127,6 @@ async function transformCode(
   asyncRequireModulePath: string,
   dynamicDepsInPackages: DynamicRequiresBehavior,
 ): Promise<Data> {
-  if (sourceCode == null) {
-    sourceCode = fs.readFileSync(filename, 'utf8');
-  }
-
   const transformFileStartLogEntry = {
     action_name: 'Transforming file',
     action_phase: 'start',
@@ -139,9 +135,16 @@ async function transformCode(
     start_timestamp: process.hrtime(),
   };
 
+  let data;
+
+  if (sourceCode == null) {
+    data = fs.readFileSync(filename);
+    sourceCode = data.toString('utf8');
+  }
+
   const sha1 = crypto
     .createHash('sha1')
-    .update(sourceCode)
+    .update(data || sourceCode)
     .digest('hex');
 
   if (filename.endsWith('.json')) {