From 305669e716e23b36138a16afa24adb4d85c0f474 Mon Sep 17 00:00:00 2001 From: Miguel Jimenez Esun Date: Thu, 12 Apr 2018 15:18:43 -0700 Subject: [PATCH] Add HTTP store Reviewed By: jeanlauliac Differential Revision: D7382414 fbshipit-source-id: 2c6cdb9330137718f49771bafdabe2b6d24e7ba8 --- packages/metro-cache/src/index.js | 2 + packages/metro-cache/src/stores/HttpStore.js | 146 +++++++++++++ .../src/stores/__tests__/HttpStore-test.js | 202 ++++++++++++++++++ packages/metro/src/Bundler.js | 1 + packages/metro/src/JSTransformer/worker.js | 13 +- 5 files changed, 359 insertions(+), 5 deletions(-) create mode 100644 packages/metro-cache/src/stores/HttpStore.js create mode 100644 packages/metro-cache/src/stores/__tests__/HttpStore-test.js diff --git a/packages/metro-cache/src/index.js b/packages/metro-cache/src/index.js index b94f6026..c91b84b8 100644 --- a/packages/metro-cache/src/index.js +++ b/packages/metro-cache/src/index.js @@ -12,6 +12,7 @@ const Cache = require('./Cache'); const FileStore = require('./stores/FileStore'); +const HttpStore = require('./stores/HttpStore'); const PersistedMapStore = require('./stores/PersistedMapStore'); const stableHash = require('./stableHash'); @@ -20,6 +21,7 @@ export type {CacheStore} from './types.flow'; module.exports.Cache = Cache; module.exports.FileStore = FileStore; +module.exports.HttpStore = HttpStore; module.exports.PersistedMapStore = PersistedMapStore; module.exports.stableHash = stableHash; diff --git a/packages/metro-cache/src/stores/HttpStore.js b/packages/metro-cache/src/stores/HttpStore.js new file mode 100644 index 00000000..13636845 --- /dev/null +++ b/packages/metro-cache/src/stores/HttpStore.js @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2018-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow + */ + +'use strict'; + +const http = require('http'); +const https = require('https'); +const url = require('url'); +const zlib = require('zlib'); + +import type {TransformedCode} from 'metro/src/JSTransformer/worker'; + +export type Options = {| + endpoint: string, + timeout?: number, +|}; + +const ZLIB_OPTIONS = { + level: 9, +}; + +class HttpStore { + _module: typeof http | typeof https; + _timeout: number; + + _host: string; + _port: number; + _path: string; + + _getAgent: http$Agent; + _setAgent: http$Agent; + + constructor(options: Options) { + const uri = url.parse(options.endpoint); + const module = uri.protocol === 'http:' ? http : https; + + const agentConfig = { + keepAlive: true, + keepAliveMsecs: options.timeout || 5000, + maxSockets: 64, + maxFreeSockets: 64, + }; + + if (!uri.hostname || !uri.pathname) { + throw new TypeError('Invalid endpoint: ' + options.endpoint); + } + + this._module = module; + this._timeout = options.timeout || 5000; + + this._host = uri.hostname; + this._path = uri.pathname; + this._port = +uri.port; + + this._getAgent = new module.Agent(agentConfig); + this._setAgent = new module.Agent(agentConfig); + } + + get(key: Buffer): Promise { + return new Promise((resolve, reject) => { + const options = { + agent: this._getAgent, + host: this._host, + method: 'GET', + path: this._path + '/' + key.toString('hex'), + port: this._port, + timeout: this._timeout, + }; + + const req = this._module.request(options, res => { + let data = ''; + + if (res.statusCode === 404) { + resolve(null); + return; + } else if (res.statusCode !== 200) { + reject(new Error('HTTP error: ' + res.statusCode)); + return; + } + + const gunzipped = res.pipe(zlib.createGunzip()); + + gunzipped.on('data', chunk => { + data += chunk.toString(); + }); + + gunzipped.on('error', err => { + reject(err); + }); + + gunzipped.on('end', () => { + try { + resolve(JSON.parse(data)); + } catch (err) { + reject(err); + } + }); + + res.on('error', err => gunzipped.emit('error', err)); + }); + + req.end(); + }); + } + + set(key: Buffer, value: TransformedCode): Promise { + return new Promise((resolve, reject) => { + const gzip = zlib.createGzip(ZLIB_OPTIONS); + + const options = { + agent: this._setAgent, + host: this._host, + method: 'PUT', + path: this._path + '/' + key.toString('hex'), + port: this._port, + timeout: this._timeout, + }; + + const req = this._module.request(options, res => { + res.on('error', err => { + reject(err); + }); + + res.on('data', () => { + // Do nothing. It is needed so node thinks we are consuming responses. + }); + + res.on('end', () => { + resolve(); + }); + }); + + gzip.pipe(req); + gzip.end(JSON.stringify(value) || 'null'); + }); + } +} + +module.exports = HttpStore; diff --git a/packages/metro-cache/src/stores/__tests__/HttpStore-test.js b/packages/metro-cache/src/stores/__tests__/HttpStore-test.js new file mode 100644 index 00000000..7e82cb4e --- /dev/null +++ b/packages/metro-cache/src/stores/__tests__/HttpStore-test.js @@ -0,0 +1,202 @@ +/** + * Copyright (c) 2018-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+javascript_foundation + * @format + */ + +'use strict'; + +const zlib = require('zlib'); + +const {PassThrough} = require('stream'); + +describe('HttpStore', () => { + let HttpStore; + let httpPassThrough; + + function responseHttpOk(data) { + const res = Object.assign(new PassThrough(), { + statusCode: 200, + }); + + process.nextTick(() => { + res.write(zlib.gzipSync(data)); + res.end(); + }); + + return res; + } + + function responseHttpError(code) { + return Object.assign(new PassThrough(), { + statusCode: code, + }); + } + + function responseError(err) { + const res = Object.assign(new PassThrough(), { + statusCode: 200, + }); + + process.nextTick(() => { + res.emit('error', err); + }); + + return res; + } + + beforeEach(() => { + jest + .resetModules() + .resetAllMocks() + .useFakeTimers() + .mock('http') + .mock('https'); + + httpPassThrough = new PassThrough(); + require('http').request.mockReturnValue(httpPassThrough); + require('https').request.mockReturnValue(httpPassThrough); + + HttpStore = require('../HttpStore'); + }); + + it('works with HTTP and HTTPS', () => { + const httpStore = new HttpStore({endpoint: 'http://example.com'}); + const httpsStore = new HttpStore({endpoint: 'https://example.com'}); + + httpStore.get(Buffer.from('foo')); + expect(require('http').request).toHaveBeenCalledTimes(1); + expect(require('https').request).not.toHaveBeenCalled(); + + jest.resetAllMocks(); + + httpsStore.get(Buffer.from('foo')); + expect(require('http').request).not.toHaveBeenCalled(); + expect(require('https').request).toHaveBeenCalledTimes(1); + }); + + it('gets using the network via GET method', async () => { + const store = new HttpStore({endpoint: 'http://www.example.com/endpoint'}); + const promise = store.get(Buffer.from('key')); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('GET'); + expect(opts.host).toEqual('www.example.com'); + expect(opts.path).toEqual('/endpoint/6b6579'); + expect(opts.timeout).toEqual(5000); + + callback(responseHttpOk(JSON.stringify({foo: 42}))); + jest.runAllTimers(); + + expect(await promise).toEqual({foo: 42}); + }); + + it('resolves with "null" when HTTP 404 is returned', async () => { + const store = new HttpStore({endpoint: 'http://example.com'}); + const promise = store.get(Buffer.from('key')); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('GET'); + + callback(responseHttpError(404)); + jest.runAllTimers(); + + expect(await promise).toEqual(null); + }); + + it('rejects when an HTTP different of 200/404 is returned', done => { + const store = new HttpStore({endpoint: 'http://example.com'}); + const promise = store.get(Buffer.from('key')); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('GET'); + + callback(responseHttpError(503)); // Intentionally unterminated JSON. + jest.runAllTimers(); + + promise.catch(err => { + expect(err).toBeInstanceOf(Error); + expect(err.message).toMatch(/HTTP error: 503/); + done(); + }); + }); + + it('rejects when it gets an invalid JSON response', done => { + const store = new HttpStore({endpoint: 'http://example.com'}); + const promise = store.get(Buffer.from('key')); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('GET'); + + callback(responseHttpOk('{"foo": 4')); // Intentionally unterminated JSON. + jest.runAllTimers(); + + promise.catch(err => { + expect(err).toBeInstanceOf(SyntaxError); + done(); + }); + }); + + it('rejects when the HTTP layer throws', done => { + const store = new HttpStore({endpoint: 'http://example.com'}); + const promise = store.get(Buffer.from('key')); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('GET'); + + callback(responseError(new Error('ENOTFOUND'))); + jest.runAllTimers(); + + promise.catch(err => { + expect(err).toBeInstanceOf(Error); + expect(err.message).toBe('ENOTFOUND'); + done(); + }); + }); + + it('sets using the network via PUT method', done => { + const store = new HttpStore({endpoint: 'http://www.example.com/endpoint'}); + const promise = store.set(Buffer.from('key-set'), {foo: 42}); + const [opts, callback] = require('http').request.mock.calls[0]; + const buf = []; + + expect(opts.method).toEqual('PUT'); + expect(opts.host).toEqual('www.example.com'); + expect(opts.path).toEqual('/endpoint/6b65792d736574'); + expect(opts.timeout).toEqual(5000); + + callback(responseHttpOk('')); + + httpPassThrough.on('data', chunk => { + buf.push(chunk); + }); + + httpPassThrough.on('end', async () => { + expect(zlib.gunzipSync(Buffer.concat(buf)).toString()).toBe('{"foo":42}'); + await promise; // Ensure that the setting promise successfully finishes. + + done(); + }); + }); + + it('rejects when setting and HTTP fails', done => { + const store = new HttpStore({endpoint: 'http://example.com'}); + const promise = store.set(Buffer.from('key-set'), {foo: 42}); + const [opts, callback] = require('http').request.mock.calls[0]; + + expect(opts.method).toEqual('PUT'); + + callback(responseError(new Error('ENOTFOUND'))); + jest.runAllTimers(); + + promise.catch(err => { + expect(err).toBeInstanceOf(Error); + expect(err.message).toBe('ENOTFOUND'); + done(); + }); + }); +}); diff --git a/packages/metro/src/Bundler.js b/packages/metro/src/Bundler.js index 12bcee7b..51e64252 100644 --- a/packages/metro/src/Bundler.js +++ b/packages/metro/src/Bundler.js @@ -177,6 +177,7 @@ class Bundler { opts.assetExts, opts.assetRegistryPath, getTransformCacheKey, + 'experimental', ]).toString('binary'); this._projectRoots = opts.projectRoots; diff --git a/packages/metro/src/JSTransformer/worker.js b/packages/metro/src/JSTransformer/worker.js index 7431ea1c..309b8411 100644 --- a/packages/metro/src/JSTransformer/worker.js +++ b/packages/metro/src/JSTransformer/worker.js @@ -127,10 +127,6 @@ async function transformCode( asyncRequireModulePath: string, dynamicDepsInPackages: DynamicRequiresBehavior, ): Promise { - if (sourceCode == null) { - sourceCode = fs.readFileSync(filename, 'utf8'); - } - const transformFileStartLogEntry = { action_name: 'Transforming file', action_phase: 'start', @@ -139,9 +135,16 @@ async function transformCode( start_timestamp: process.hrtime(), }; + let data; + + if (sourceCode == null) { + data = fs.readFileSync(filename); + sourceCode = data.toString('utf8'); + } + const sha1 = crypto .createHash('sha1') - .update(sourceCode) + .update(data || sourceCode) .digest('hex'); if (filename.endsWith('.json')) {