diff --git a/react-packager/__mocks__/net.js b/react-packager/__mocks__/net.js deleted file mode 100644 index 43f51828..00000000 --- a/react-packager/__mocks__/net.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright (c) 2015-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -var EventEmitter = require('events').EventEmitter; -var servers = {}; -exports.createServer = function(listener) { - var server = { - _listener: listener, - - socket: new EventEmitter(), - - listen: function(path) { - listener(this.socket); - servers[path] = this; - } - }; - - server.socket.setEncoding = function() {}; - server.socket.write = function(data) { - this.emit('data', data); - }; - - return server; -}; - -exports.connect = function(options) { - var server = servers[options.path || options.port]; - return server.socket; -}; diff --git a/react-packager/index.js b/react-packager/index.js index 21bb1d67..c3e5829f 100644 --- a/react-packager/index.js +++ b/react-packager/index.js @@ -16,6 +16,7 @@ useGracefulFs(); var Activity = require('./src/Activity'); var Server = require('./src/Server'); +var SocketInterface = require('./src/SocketInterface'); exports.middleware = function(options) { var server = new Server(options); @@ -55,6 +56,40 @@ exports.getDependencies = function(options, main) { }); }; +exports.createClientFor = function(options) { + return SocketInterface.getOrCreateSocketFor(options); +}; + +process.on('message', function(m) { + if (m && m.type && m.type === 'createSocketServer') { + console.log('server got ipc message', m); + var options = m.data.options; + + // regexp doesn't naturally serialize to json. + options.blacklistRE = new RegExp(options.blacklistRE.source); + + SocketInterface.createSocketServer( + m.data.sockPath, + m.data.options + ).then( + function() { + console.log('succesfully created server', m); + process.send({ type: 'createdServer' }); + }, + function(error) { + console.log('error creating server', error.code); + if (error.code === 'EADDRINUSE') { + // Server already listening, this may happen if multiple + // clients where started in quick succussion (buck). + process.send({ type: 'createdServer' }); + } else { + throw error; + } + } + ).done(); + } +}); + function useGracefulFs() { var fs = require('fs'); var gracefulFs = require('graceful-fs'); diff --git a/react-packager/src/Bundler/Bundle.js b/react-packager/src/Bundler/Bundle.js index 418f7a9e..b7920ebb 100644 --- a/react-packager/src/Bundler/Bundle.js +++ b/react-packager/src/Bundler/Bundle.js @@ -284,6 +284,36 @@ class Bundle { }).join('\n'), ].join('\n'); } + + toJSON() { + if (!this._finalized) { + throw new Error('Cannot serialize bundle unless finalized'); + } + + return { + modules: this._modules, + assets: this._assets, + sourceMapUrl: this._sourceMapUrl, + shouldCombineSourceMaps: this._shouldCombineSourceMaps, + mainModuleId: this._mainModuleId, + }; + } + + static fromJSON(json) { + const bundle = new Bundle(json.sourceMapUrl); + bundle._mainModuleId = json.mainModuleId; + bundle._assets = json.assets; + bundle._modules = json.modules; + bundle._sourceMapUrl = json.sourceMapUrl; + + Object.freeze(bundle._modules); + Object.seal(bundle._modules); + Object.freeze(bundle._assets); + Object.seal(bundle._assets); + bundle._finalized = true; + + return bundle; + } } function generateSourceMapForVirtualModule(module) { diff --git a/react-packager/src/Bundler/index.js b/react-packager/src/Bundler/index.js index c1d811d4..d29ab0f8 100644 --- a/react-packager/src/Bundler/index.js +++ b/react-packager/src/Bundler/index.js @@ -86,20 +86,17 @@ class Bundler { opts.projectRoots.forEach(verifyRootExists); - this._cache = opts.nonPersistent - ? new DummyCache() - : new Cache({ - resetCache: opts.resetCache, - cacheVersion: opts.cacheVersion, - projectRoots: opts.projectRoots, - transformModulePath: opts.transformModulePath, - }); + this._cache = new Cache({ + resetCache: opts.resetCache, + cacheVersion: opts.cacheVersion, + projectRoots: opts.projectRoots, + transformModulePath: opts.transformModulePath, + }); this._resolver = new DependencyResolver({ projectRoots: opts.projectRoots, blacklistRE: opts.blacklistRE, polyfillModuleNames: opts.polyfillModuleNames, - nonPersistent: opts.nonPersistent, moduleFormat: opts.moduleFormat, assetRoots: opts.assetRoots, fileWatcher: opts.fileWatcher, diff --git a/react-packager/src/Cache/index.js b/react-packager/src/Cache/index.js index 2ed3575e..ae6d1aa3 100644 --- a/react-packager/src/Cache/index.js +++ b/react-packager/src/Cache/index.js @@ -215,6 +215,7 @@ class Cache { hash.update(options.transformModulePath); var name = 'react-packager-cache-' + hash.digest('hex'); + return path.join(tmpdir, name); } } diff --git a/react-packager/src/DependencyResolver/Module.js b/react-packager/src/DependencyResolver/Module.js index 72584f8f..745ab3fb 100644 --- a/react-packager/src/DependencyResolver/Module.js +++ b/react-packager/src/DependencyResolver/Module.js @@ -8,7 +8,6 @@ */ 'use strict'; -const Promise = require('promise'); const docblock = require('./DependencyGraph/docblock'); const isAbsolutePath = require('absolute-path'); const path = require('path'); @@ -128,6 +127,17 @@ class Module { isAsset_DEPRECATED() { return false; } + + toJSON() { + return { + hash: this.hash(), + isJSON: this.isJSON(), + isAsset: this.isAsset(), + isAsset_DEPRECATED: this.isAsset_DEPRECATED(), + type: this.type, + path: this.path, + }; + } } /** diff --git a/react-packager/src/DependencyResolver/index.js b/react-packager/src/DependencyResolver/index.js index 329ae8b7..cc7f9468 100644 --- a/react-packager/src/DependencyResolver/index.js +++ b/react-packager/src/DependencyResolver/index.js @@ -27,10 +27,6 @@ var validateOpts = declareOpts({ type: 'array', default: [], }, - nonPersistent: { - type: 'boolean', - default: false, - }, moduleFormat: { type: 'string', default: 'haste', diff --git a/react-packager/src/SocketInterface/SocketClient.js b/react-packager/src/SocketInterface/SocketClient.js new file mode 100644 index 00000000..474223ad --- /dev/null +++ b/react-packager/src/SocketInterface/SocketClient.js @@ -0,0 +1,101 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +const Bundle = require('../Bundler/Bundle'); +const Promise = require('promise'); +const bser = require('bser'); +const debug = require('debug')('ReactPackager:SocketClient'); +const net = require('net'); + +class SocketClient { + static create(sockPath) { + return new SocketClient(sockPath).onReady(); + } + + constructor(sockPath) { + debug('connecting to', sockPath); + + this._sock = net.connect(sockPath); + this._ready = new Promise((resolve, reject) => { + this._sock.on('connect', () => resolve(this)); + this._sock.on('error', (e) => reject(e)); + }); + + this._resolvers = Object.create(null); + const bunser = new bser.BunserBuf(); + this._sock.on('data', (buf) => bunser.append(buf)); + + bunser.on('value', (message) => this._handleMessage(message)); + } + + onReady() { + return this._ready; + } + + getDependencies(main) { + return this._send({ + type: 'getDependencies', + data: main, + }); + } + + buildBundle(options) { + return this._send({ + type: 'buildBundle', + data: options, + }).then(json => Bundle.fromJSON(json)); + } + + _send(message) { + message.id = uid(); + this._sock.write(bser.dumpToBuffer(message)); + return new Promise((resolve, reject) => { + this._resolvers[message.id] = {resolve, reject}; + }); + } + + _handleMessage(message) { + if (!(message && message.id && message.type)) { + throw new Error( + 'Malformed message from server ' + JSON.stringify(message) + ); + } + + debug('got message with type', message.type); + + const resolver = this._resolvers[message.id]; + if (!resolver) { + throw new Error( + 'Unrecognized message id (message already resolved or never existed' + ); + } + + delete this._resolvers[message.id]; + + if (message.type === 'error') { + // TODO convert to an error + resolver.reject(message.data); + } else { + resolver.resolve(message.data); + } + } + + close() { + debug('closing connection'); + this._sock.end(); + } +} + +module.exports = SocketClient; + +function uid(len) { + len = len || 7; + return Math.random().toString(35).substr(2, len); +} diff --git a/react-packager/src/SocketInterface/SocketServer.js b/react-packager/src/SocketInterface/SocketServer.js new file mode 100644 index 00000000..693c8e0c --- /dev/null +++ b/react-packager/src/SocketInterface/SocketServer.js @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +const Promise = require('promise'); +const Server = require('../Server'); +const bser = require('bser'); +const debug = require('debug')('ReactPackager:SocketServer'); +const fs = require('fs'); +const net = require('net'); + +const MAX_IDLE_TIME = 10 * 60 * 1000; + +class SocketServer { + constructor(sockPath, options) { + this._server = net.createServer(); + this._server.listen(sockPath); + this._ready = new Promise((resolve, reject) => { + this._server.on('error', (e) => reject(e)); + this._server.on('listening', () => { + debug( + 'Process %d listening on socket path %s ' + + 'for server with options %j', + process.pid, + sockPath, + options + ); + resolve(this); + }); + }); + this._server.on('connection', (sock) => this._handleConnection(sock)); + + // Disable the file watcher. + options.nonPersistent = true; + this._packagerServer = new Server(options); + this._jobs = 0; + this._dieEventually(); + + process.on('exit', () => fs.unlinkSync(sockPath)); + } + + onReady() { + return this._ready; + } + + _handleConnection(sock) { + debug('connection to server', process.pid); + + const bunser = new bser.BunserBuf(); + sock.on('data', (buf) => bunser.append(buf)); + + bunser.on('value', (m) => this._handleMessage(sock, m)); + } + + _handleMessage(sock, m) { + if (!m || !m.id || !m.data) { + console.error('SocketServer recieved a malformed message: %j', m); + } + + debug('got request', m); + + // Debounce the kill timer. + this._dieEventually(); + + const handleError = (error) => { + debug('request error', error); + this._jobs--; + this._reply(sock, m.id, 'error', error.stack); + }; + + switch (m.type) { + case 'getDependencies': + this._jobs++; + this._packagerServer.getDependencies(m.data).then( + ({ dependencies }) => this._reply(sock, m.id, 'result', dependencies), + handleError, + ); + break; + + case 'buildBundle': + this._jobs++; + this._packagerServer.buildBundle(m.data).then( + (result) => this._reply(sock, m.id, 'result', result), + handleError, + ); + break; + + default: + this._reply(sock, m.id, 'error', 'Unknown message type: ' + m.type); + } + } + + _reply(sock, id, type, data) { + debug('request finished', type); + + this._jobs--; + data = toJSON(data); + + sock.write(bser.dumpToBuffer({ + id, + type, + data, + })); + } + + _dieEventually() { + clearTimeout(this._deathTimer); + this._deathTimer = setTimeout(() => { + if (this._jobs <= 0) { + debug('server dying', process.pid); + process.exit(1); + } + this._dieEventually(); + }, MAX_IDLE_TIME); + } +} + +module.exports = SocketServer; + +// TODO move this to bser code. +function toJSON(object) { + if (!(object && typeof object === 'object')) { + return object; + } + + if (object.toJSON) { + return object.toJSON(); + } + + for (var p in object) { + object[p] = toJSON(object[p]); + } + + return object; +} diff --git a/react-packager/src/SocketInterface/__tests__/SocketClient-test.js b/react-packager/src/SocketInterface/__tests__/SocketClient-test.js new file mode 100644 index 00000000..b77d92a0 --- /dev/null +++ b/react-packager/src/SocketInterface/__tests__/SocketClient-test.js @@ -0,0 +1,112 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +jest.setMock('worker-farm', function() { return () => {}; }) + .setMock('uglify-js') + .mock('net') + .dontMock('../SocketClient'); + +describe('SocketClient', () => { + let SocketClient; + let sock; + let bunser; + + beforeEach(() => { + SocketClient = require('../SocketClient'); + + const {EventEmitter} = require.requireActual('events'); + sock = new EventEmitter(); + sock.write = jest.genMockFn(); + + require('net').connect.mockImpl(() => sock); + + const bser = require('bser'); + bunser = new EventEmitter(); + require('bser').BunserBuf.mockImpl(() => bunser); + bser.dumpToBuffer.mockImpl((a) => a); + + require('../../Bundler/Bundle').fromJSON.mockImpl((a) => a); + }); + + pit('create a connection', () => { + const client = new SocketClient('/sock'); + sock.emit('connect'); + return client.onReady().then(c => { + expect(c).toBe(client); + expect(require('net').connect).toBeCalledWith('/sock'); + }); + }); + + pit('buildBundle', () => { + const client = new SocketClient('/sock'); + sock.emit('connect'); + const options = { entryFile: '/main' }; + + const promise = client.buildBundle(options); + + expect(sock.write).toBeCalled(); + const message = sock.write.mock.calls[0][0]; + expect(message.type).toBe('buildBundle'); + expect(message.data).toEqual(options); + expect(typeof message.id).toBe('string'); + + bunser.emit('value', { + id: message.id, + type: 'result', + data: { bundle: 'foo' }, + }); + + return promise.then(bundle => expect(bundle).toEqual({ bundle: 'foo' })); + }); + + pit('getDependencies', () => { + const client = new SocketClient('/sock'); + sock.emit('connect'); + const main = '/main'; + + const promise = client.getDependencies(main); + + expect(sock.write).toBeCalled(); + const message = sock.write.mock.calls[0][0]; + expect(message.type).toBe('getDependencies'); + expect(message.data).toEqual(main); + expect(typeof message.id).toBe('string'); + + bunser.emit('value', { + id: message.id, + type: 'result', + data: ['a', 'b', 'c'], + }); + + return promise.then(result => expect(result).toEqual(['a', 'b', 'c'])); + }); + + pit('handle errors', () => { + const client = new SocketClient('/sock'); + sock.emit('connect'); + const main = '/main'; + + const promise = client.getDependencies(main); + + expect(sock.write).toBeCalled(); + const message = sock.write.mock.calls[0][0]; + expect(message.type).toBe('getDependencies'); + expect(message.data).toEqual(main); + expect(typeof message.id).toBe('string'); + + bunser.emit('value', { + id: message.id, + type: 'error', + data: 'some error' + }); + + return promise.catch(m => expect(m).toBe('some error')); + }); +}); diff --git a/react-packager/src/SocketInterface/__tests__/SocketInterface-test.js b/react-packager/src/SocketInterface/__tests__/SocketInterface-test.js new file mode 100644 index 00000000..f0940023 --- /dev/null +++ b/react-packager/src/SocketInterface/__tests__/SocketInterface-test.js @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +jest.setMock('worker-farm', function() { return () => {}; }) + .setMock('uglify-js') + .mock('child_process') + .dontMock('../'); + +describe('SocketInterface', () => { + let SocketInterface; + let SocketClient; + + beforeEach(() => { + SocketInterface = require('../'); + SocketClient = require('../SocketClient'); + }); + + describe('getOrCreateSocketFor', () => { + pit('creates socket path by hashing options', () => { + const fs = require('fs'); + fs.existsSync = jest.genMockFn().mockImpl(() => true); + + // Check that given two equivelant server options, we end up with the same + // socket path. + const options1 = { projectRoots: ['/root'], transformModulePath: '/root/foo' }; + const options2 = { transformModulePath: '/root/foo', projectRoots: ['/root'] }; + const options3 = { projectRoots: ['/root', '/root2'] }; + + return SocketInterface.getOrCreateSocketFor(options1).then(() => { + expect(SocketClient.create).toBeCalled(); + return SocketInterface.getOrCreateSocketFor(options2).then(() => { + expect(SocketClient.create.mock.calls.length).toBe(2); + expect(SocketClient.create.mock.calls[0]).toEqual(SocketClient.create.mock.calls[1]); + return SocketInterface.getOrCreateSocketFor(options3).then(() => { + expect(SocketClient.create.mock.calls.length).toBe(3); + expect(SocketClient.create.mock.calls[1]).not.toEqual(SocketClient.create.mock.calls[2]); + }); + }); + }); + }); + + pit('should fork a server', () => { + const fs = require('fs'); + fs.existsSync = jest.genMockFn().mockImpl(() => false); + let sockPath; + let callback; + + require('child_process').spawn.mockImpl(() => ({ + on: (event, cb) => callback = cb, + send: (message) => { + expect(message.type).toBe('createSocketServer'); + expect(message.data.options).toEqual({ projectRoots: ['/root'] }); + expect(message.data.sockPath).toContain('react-packager'); + sockPath = message.data.sockPath; + + setImmediate(() => callback({ type: 'createdServer' })); + }, + unref: () => undefined, + disconnect: () => undefined, + })); + + return SocketInterface.getOrCreateSocketFor({ projectRoots: ['/root'] }) + .then(() => { + expect(SocketClient.create).toBeCalledWith(sockPath); + }); + }); + }); + + describe('createSocketServer', () => { + pit('creates a server', () => { + require('../SocketServer').mockImpl((sockPath, options) => { + expect(sockPath).toBe('/socket'); + expect(options).toEqual({ projectRoots: ['/root'] }); + return { onReady: () => Promise.resolve() }; + }); + + return SocketInterface.createSocketServer('/socket', { projectRoots: ['/root'] }); + }); + }); +}); diff --git a/react-packager/src/SocketInterface/__tests__/SocketServer-test.js b/react-packager/src/SocketInterface/__tests__/SocketServer-test.js new file mode 100644 index 00000000..ae3e8e24 --- /dev/null +++ b/react-packager/src/SocketInterface/__tests__/SocketServer-test.js @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +jest.setMock('worker-farm', function() { return () => {}; }) + .setMock('uglify-js') + .mock('net') + .mock('fs') + .dontMock('../SocketServer'); + +describe('SocketServer', () => { + let PackagerServer; + let SocketServer; + let netServer; + let bunser; + + beforeEach(() => { + SocketServer = require('../SocketServer'); + + const {EventEmitter} = require.requireActual('events'); + netServer = new EventEmitter(); + netServer.listen = jest.genMockFn(); + require('net').createServer.mockImpl(() => netServer); + + const bser = require('bser'); + bunser = new EventEmitter(); + bser.BunserBuf.mockImpl(() => bunser); + bser.dumpToBuffer.mockImpl((a) => a); + + PackagerServer = require('../../Server'); + }); + + pit('create a server', () => { + const server = new SocketServer('/sock', { projectRoots: ['/root'] }); + netServer.emit('listening'); + return server.onReady().then(s => { + expect(s).toBe(server); + expect(netServer.listen).toBeCalledWith('/sock'); + }); + }); + + pit('handles getDependencies message', () => { + const server = new SocketServer('/sock', { projectRoots: ['/root'] }); + netServer.emit('listening'); + return server.onReady().then(() => { + const sock = { on: jest.genMockFn(), write: jest.genMockFn() }; + netServer.emit('connection', sock); + PackagerServer.prototype.getDependencies.mockImpl( + () => Promise.resolve({ dependencies: ['a', 'b', 'c'] }) + ); + bunser.emit('value', { type: 'getDependencies', id: 1, data: '/main' }); + expect(PackagerServer.prototype.getDependencies).toBeCalledWith('/main'); + + // Run pending promises. + return Promise.resolve().then(() => { + expect(sock.write).toBeCalledWith( + { id: 1, type: 'result', data: ['a', 'b', 'c']} + ); + }); + }); + }); + + pit('handles buildBundle message', () => { + const server = new SocketServer('/sock', { projectRoots: ['/root'] }); + netServer.emit('listening'); + return server.onReady().then(() => { + const sock = { on: jest.genMockFn(), write: jest.genMockFn() }; + netServer.emit('connection', sock); + PackagerServer.prototype.buildBundle.mockImpl( + () => Promise.resolve({ bundle: 'foo' }) + ); + bunser.emit( + 'value', + { type: 'buildBundle', id: 1, data: { options: 'bar' } } + ); + expect(PackagerServer.prototype.buildBundle).toBeCalledWith( + { options: 'bar' } + ); + + // Run pending promises. + return Promise.resolve().then(() => { + expect(sock.write).toBeCalledWith( + { id: 1, type: 'result', data: { bundle: 'foo' }} + ); + }); + }); + }); +}); diff --git a/react-packager/src/SocketInterface/index.js b/react-packager/src/SocketInterface/index.js new file mode 100644 index 00000000..87837b29 --- /dev/null +++ b/react-packager/src/SocketInterface/index.js @@ -0,0 +1,102 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +'use strict'; + +const Promise = require('promise'); +const SocketClient = require('./SocketClient'); +const SocketServer = require('./SocketServer'); +const _ = require('underscore'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const tmpdir = require('os').tmpdir(); +const {spawn} = require('child_process'); + +const CREATE_SERVER_TIMEOUT = 10000; + +const SocketInterface = { + getOrCreateSocketFor(options) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash('md5'); + Object.keys(options).sort().forEach(key => { + if (options[key] && typeof options[key] !== 'string') { + hash.update(JSON.stringify(options[key])); + } else { + hash.update(options[key]); + } + }); + + const sockPath = path.join( + tmpdir, + 'react-packager-' + hash.digest('hex') + ); + + if (fs.existsSync(sockPath)) { + resolve(SocketClient.create(sockPath)); + return; + } + + const logPath = path.join(tmpdir, 'react-packager.log'); + + const timeout = setTimeout( + () => reject( + new Error( + 'Took too long to start server. Server logs: \n' + + fs.readFileSync(logPath, 'utf8') + ) + ), + CREATE_SERVER_TIMEOUT, + ); + + const log = fs.openSync(logPath, 'a'); + + // Enable server debugging by default since it's going to a log file. + const env = _.clone(process.env); + env.DEBUG = 'ReactPackager:SocketServer'; + + // We have to go through the main entry point to make sure + // we go through the babel require hook. + const child = spawn( + process.execPath, + [path.join(__dirname, '..', '..', 'index.js')], + { + detached: true, + env: env, + stdio: ['ipc', log, log] + } + ); + + child.unref(); + + child.on('message', m => { + if (m && m.type && m.type === 'createdServer') { + clearTimeout(timeout); + child.disconnect(); + resolve(SocketClient.create(sockPath)); + } + }); + + + if (options.blacklistRE) { + options.blacklistRE = { source: options.blacklistRE.source }; + } + + child.send({ + type: 'createSocketServer', + data: { sockPath, options } + }); + }); + }, + + createSocketServer(sockPath, options) { + return new SocketServer(sockPath, options).onReady(); + } +}; + +module.exports = SocketInterface;