[react-packager] Expose a socket interface

Summary:
Buck (our build system) currently starts multiple packager instances for each target and may build multiple targets in parallel. This means we're paying startup costs and are duplicating the work. This enables us to start one instance of the packager and connect to it via socket to do all the work that needs to be done.

The way this is structured:

1. SocketServer: A server that listens on a socket path that is generated based on the server options
2. SocketClient: Interfaces with the server and exposes the operations that we support as methods
3. SocketInterface: Integration point and responsible for forking off the server
This commit is contained in:
Amjad Masad 2015-08-25 09:47:49 -07:00
parent 81d5a70806
commit 54125e524a
13 changed files with 720 additions and 49 deletions

View File

@ -1,35 +0,0 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
var EventEmitter = require('events').EventEmitter;
var servers = {};
exports.createServer = function(listener) {
var server = {
_listener: listener,
socket: new EventEmitter(),
listen: function(path) {
listener(this.socket);
servers[path] = this;
}
};
server.socket.setEncoding = function() {};
server.socket.write = function(data) {
this.emit('data', data);
};
return server;
};
exports.connect = function(options) {
var server = servers[options.path || options.port];
return server.socket;
};

View File

@ -16,6 +16,7 @@ useGracefulFs();
var Activity = require('./src/Activity');
var Server = require('./src/Server');
var SocketInterface = require('./src/SocketInterface');
exports.middleware = function(options) {
var server = new Server(options);
@ -55,6 +56,40 @@ exports.getDependencies = function(options, main) {
});
};
exports.createClientFor = function(options) {
return SocketInterface.getOrCreateSocketFor(options);
};
process.on('message', function(m) {
if (m && m.type && m.type === 'createSocketServer') {
console.log('server got ipc message', m);
var options = m.data.options;
// regexp doesn't naturally serialize to json.
options.blacklistRE = new RegExp(options.blacklistRE.source);
SocketInterface.createSocketServer(
m.data.sockPath,
m.data.options
).then(
function() {
console.log('succesfully created server', m);
process.send({ type: 'createdServer' });
},
function(error) {
console.log('error creating server', error.code);
if (error.code === 'EADDRINUSE') {
// Server already listening, this may happen if multiple
// clients where started in quick succussion (buck).
process.send({ type: 'createdServer' });
} else {
throw error;
}
}
).done();
}
});
function useGracefulFs() {
var fs = require('fs');
var gracefulFs = require('graceful-fs');

View File

@ -284,6 +284,36 @@ class Bundle {
}).join('\n'),
].join('\n');
}
toJSON() {
if (!this._finalized) {
throw new Error('Cannot serialize bundle unless finalized');
}
return {
modules: this._modules,
assets: this._assets,
sourceMapUrl: this._sourceMapUrl,
shouldCombineSourceMaps: this._shouldCombineSourceMaps,
mainModuleId: this._mainModuleId,
};
}
static fromJSON(json) {
const bundle = new Bundle(json.sourceMapUrl);
bundle._mainModuleId = json.mainModuleId;
bundle._assets = json.assets;
bundle._modules = json.modules;
bundle._sourceMapUrl = json.sourceMapUrl;
Object.freeze(bundle._modules);
Object.seal(bundle._modules);
Object.freeze(bundle._assets);
Object.seal(bundle._assets);
bundle._finalized = true;
return bundle;
}
}
function generateSourceMapForVirtualModule(module) {

View File

@ -86,20 +86,17 @@ class Bundler {
opts.projectRoots.forEach(verifyRootExists);
this._cache = opts.nonPersistent
? new DummyCache()
: new Cache({
resetCache: opts.resetCache,
cacheVersion: opts.cacheVersion,
projectRoots: opts.projectRoots,
transformModulePath: opts.transformModulePath,
});
this._cache = new Cache({
resetCache: opts.resetCache,
cacheVersion: opts.cacheVersion,
projectRoots: opts.projectRoots,
transformModulePath: opts.transformModulePath,
});
this._resolver = new DependencyResolver({
projectRoots: opts.projectRoots,
blacklistRE: opts.blacklistRE,
polyfillModuleNames: opts.polyfillModuleNames,
nonPersistent: opts.nonPersistent,
moduleFormat: opts.moduleFormat,
assetRoots: opts.assetRoots,
fileWatcher: opts.fileWatcher,

View File

@ -215,6 +215,7 @@ class Cache {
hash.update(options.transformModulePath);
var name = 'react-packager-cache-' + hash.digest('hex');
return path.join(tmpdir, name);
}
}

View File

@ -8,7 +8,6 @@
*/
'use strict';
const Promise = require('promise');
const docblock = require('./DependencyGraph/docblock');
const isAbsolutePath = require('absolute-path');
const path = require('path');
@ -128,6 +127,17 @@ class Module {
isAsset_DEPRECATED() {
return false;
}
toJSON() {
return {
hash: this.hash(),
isJSON: this.isJSON(),
isAsset: this.isAsset(),
isAsset_DEPRECATED: this.isAsset_DEPRECATED(),
type: this.type,
path: this.path,
};
}
}
/**

View File

@ -27,10 +27,6 @@ var validateOpts = declareOpts({
type: 'array',
default: [],
},
nonPersistent: {
type: 'boolean',
default: false,
},
moduleFormat: {
type: 'string',
default: 'haste',

View File

@ -0,0 +1,101 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
const Bundle = require('../Bundler/Bundle');
const Promise = require('promise');
const bser = require('bser');
const debug = require('debug')('ReactPackager:SocketClient');
const net = require('net');
class SocketClient {
static create(sockPath) {
return new SocketClient(sockPath).onReady();
}
constructor(sockPath) {
debug('connecting to', sockPath);
this._sock = net.connect(sockPath);
this._ready = new Promise((resolve, reject) => {
this._sock.on('connect', () => resolve(this));
this._sock.on('error', (e) => reject(e));
});
this._resolvers = Object.create(null);
const bunser = new bser.BunserBuf();
this._sock.on('data', (buf) => bunser.append(buf));
bunser.on('value', (message) => this._handleMessage(message));
}
onReady() {
return this._ready;
}
getDependencies(main) {
return this._send({
type: 'getDependencies',
data: main,
});
}
buildBundle(options) {
return this._send({
type: 'buildBundle',
data: options,
}).then(json => Bundle.fromJSON(json));
}
_send(message) {
message.id = uid();
this._sock.write(bser.dumpToBuffer(message));
return new Promise((resolve, reject) => {
this._resolvers[message.id] = {resolve, reject};
});
}
_handleMessage(message) {
if (!(message && message.id && message.type)) {
throw new Error(
'Malformed message from server ' + JSON.stringify(message)
);
}
debug('got message with type', message.type);
const resolver = this._resolvers[message.id];
if (!resolver) {
throw new Error(
'Unrecognized message id (message already resolved or never existed'
);
}
delete this._resolvers[message.id];
if (message.type === 'error') {
// TODO convert to an error
resolver.reject(message.data);
} else {
resolver.resolve(message.data);
}
}
close() {
debug('closing connection');
this._sock.end();
}
}
module.exports = SocketClient;
function uid(len) {
len = len || 7;
return Math.random().toString(35).substr(2, len);
}

View File

@ -0,0 +1,141 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
const Promise = require('promise');
const Server = require('../Server');
const bser = require('bser');
const debug = require('debug')('ReactPackager:SocketServer');
const fs = require('fs');
const net = require('net');
const MAX_IDLE_TIME = 10 * 60 * 1000;
class SocketServer {
constructor(sockPath, options) {
this._server = net.createServer();
this._server.listen(sockPath);
this._ready = new Promise((resolve, reject) => {
this._server.on('error', (e) => reject(e));
this._server.on('listening', () => {
debug(
'Process %d listening on socket path %s ' +
'for server with options %j',
process.pid,
sockPath,
options
);
resolve(this);
});
});
this._server.on('connection', (sock) => this._handleConnection(sock));
// Disable the file watcher.
options.nonPersistent = true;
this._packagerServer = new Server(options);
this._jobs = 0;
this._dieEventually();
process.on('exit', () => fs.unlinkSync(sockPath));
}
onReady() {
return this._ready;
}
_handleConnection(sock) {
debug('connection to server', process.pid);
const bunser = new bser.BunserBuf();
sock.on('data', (buf) => bunser.append(buf));
bunser.on('value', (m) => this._handleMessage(sock, m));
}
_handleMessage(sock, m) {
if (!m || !m.id || !m.data) {
console.error('SocketServer recieved a malformed message: %j', m);
}
debug('got request', m);
// Debounce the kill timer.
this._dieEventually();
const handleError = (error) => {
debug('request error', error);
this._jobs--;
this._reply(sock, m.id, 'error', error.stack);
};
switch (m.type) {
case 'getDependencies':
this._jobs++;
this._packagerServer.getDependencies(m.data).then(
({ dependencies }) => this._reply(sock, m.id, 'result', dependencies),
handleError,
);
break;
case 'buildBundle':
this._jobs++;
this._packagerServer.buildBundle(m.data).then(
(result) => this._reply(sock, m.id, 'result', result),
handleError,
);
break;
default:
this._reply(sock, m.id, 'error', 'Unknown message type: ' + m.type);
}
}
_reply(sock, id, type, data) {
debug('request finished', type);
this._jobs--;
data = toJSON(data);
sock.write(bser.dumpToBuffer({
id,
type,
data,
}));
}
_dieEventually() {
clearTimeout(this._deathTimer);
this._deathTimer = setTimeout(() => {
if (this._jobs <= 0) {
debug('server dying', process.pid);
process.exit(1);
}
this._dieEventually();
}, MAX_IDLE_TIME);
}
}
module.exports = SocketServer;
// TODO move this to bser code.
function toJSON(object) {
if (!(object && typeof object === 'object')) {
return object;
}
if (object.toJSON) {
return object.toJSON();
}
for (var p in object) {
object[p] = toJSON(object[p]);
}
return object;
}

View File

@ -0,0 +1,112 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.setMock('worker-farm', function() { return () => {}; })
.setMock('uglify-js')
.mock('net')
.dontMock('../SocketClient');
describe('SocketClient', () => {
let SocketClient;
let sock;
let bunser;
beforeEach(() => {
SocketClient = require('../SocketClient');
const {EventEmitter} = require.requireActual('events');
sock = new EventEmitter();
sock.write = jest.genMockFn();
require('net').connect.mockImpl(() => sock);
const bser = require('bser');
bunser = new EventEmitter();
require('bser').BunserBuf.mockImpl(() => bunser);
bser.dumpToBuffer.mockImpl((a) => a);
require('../../Bundler/Bundle').fromJSON.mockImpl((a) => a);
});
pit('create a connection', () => {
const client = new SocketClient('/sock');
sock.emit('connect');
return client.onReady().then(c => {
expect(c).toBe(client);
expect(require('net').connect).toBeCalledWith('/sock');
});
});
pit('buildBundle', () => {
const client = new SocketClient('/sock');
sock.emit('connect');
const options = { entryFile: '/main' };
const promise = client.buildBundle(options);
expect(sock.write).toBeCalled();
const message = sock.write.mock.calls[0][0];
expect(message.type).toBe('buildBundle');
expect(message.data).toEqual(options);
expect(typeof message.id).toBe('string');
bunser.emit('value', {
id: message.id,
type: 'result',
data: { bundle: 'foo' },
});
return promise.then(bundle => expect(bundle).toEqual({ bundle: 'foo' }));
});
pit('getDependencies', () => {
const client = new SocketClient('/sock');
sock.emit('connect');
const main = '/main';
const promise = client.getDependencies(main);
expect(sock.write).toBeCalled();
const message = sock.write.mock.calls[0][0];
expect(message.type).toBe('getDependencies');
expect(message.data).toEqual(main);
expect(typeof message.id).toBe('string');
bunser.emit('value', {
id: message.id,
type: 'result',
data: ['a', 'b', 'c'],
});
return promise.then(result => expect(result).toEqual(['a', 'b', 'c']));
});
pit('handle errors', () => {
const client = new SocketClient('/sock');
sock.emit('connect');
const main = '/main';
const promise = client.getDependencies(main);
expect(sock.write).toBeCalled();
const message = sock.write.mock.calls[0][0];
expect(message.type).toBe('getDependencies');
expect(message.data).toEqual(main);
expect(typeof message.id).toBe('string');
bunser.emit('value', {
id: message.id,
type: 'error',
data: 'some error'
});
return promise.catch(m => expect(m).toBe('some error'));
});
});

View File

@ -0,0 +1,87 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.setMock('worker-farm', function() { return () => {}; })
.setMock('uglify-js')
.mock('child_process')
.dontMock('../');
describe('SocketInterface', () => {
let SocketInterface;
let SocketClient;
beforeEach(() => {
SocketInterface = require('../');
SocketClient = require('../SocketClient');
});
describe('getOrCreateSocketFor', () => {
pit('creates socket path by hashing options', () => {
const fs = require('fs');
fs.existsSync = jest.genMockFn().mockImpl(() => true);
// Check that given two equivelant server options, we end up with the same
// socket path.
const options1 = { projectRoots: ['/root'], transformModulePath: '/root/foo' };
const options2 = { transformModulePath: '/root/foo', projectRoots: ['/root'] };
const options3 = { projectRoots: ['/root', '/root2'] };
return SocketInterface.getOrCreateSocketFor(options1).then(() => {
expect(SocketClient.create).toBeCalled();
return SocketInterface.getOrCreateSocketFor(options2).then(() => {
expect(SocketClient.create.mock.calls.length).toBe(2);
expect(SocketClient.create.mock.calls[0]).toEqual(SocketClient.create.mock.calls[1]);
return SocketInterface.getOrCreateSocketFor(options3).then(() => {
expect(SocketClient.create.mock.calls.length).toBe(3);
expect(SocketClient.create.mock.calls[1]).not.toEqual(SocketClient.create.mock.calls[2]);
});
});
});
});
pit('should fork a server', () => {
const fs = require('fs');
fs.existsSync = jest.genMockFn().mockImpl(() => false);
let sockPath;
let callback;
require('child_process').spawn.mockImpl(() => ({
on: (event, cb) => callback = cb,
send: (message) => {
expect(message.type).toBe('createSocketServer');
expect(message.data.options).toEqual({ projectRoots: ['/root'] });
expect(message.data.sockPath).toContain('react-packager');
sockPath = message.data.sockPath;
setImmediate(() => callback({ type: 'createdServer' }));
},
unref: () => undefined,
disconnect: () => undefined,
}));
return SocketInterface.getOrCreateSocketFor({ projectRoots: ['/root'] })
.then(() => {
expect(SocketClient.create).toBeCalledWith(sockPath);
});
});
});
describe('createSocketServer', () => {
pit('creates a server', () => {
require('../SocketServer').mockImpl((sockPath, options) => {
expect(sockPath).toBe('/socket');
expect(options).toEqual({ projectRoots: ['/root'] });
return { onReady: () => Promise.resolve() };
});
return SocketInterface.createSocketServer('/socket', { projectRoots: ['/root'] });
});
});
});

View File

@ -0,0 +1,94 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.setMock('worker-farm', function() { return () => {}; })
.setMock('uglify-js')
.mock('net')
.mock('fs')
.dontMock('../SocketServer');
describe('SocketServer', () => {
let PackagerServer;
let SocketServer;
let netServer;
let bunser;
beforeEach(() => {
SocketServer = require('../SocketServer');
const {EventEmitter} = require.requireActual('events');
netServer = new EventEmitter();
netServer.listen = jest.genMockFn();
require('net').createServer.mockImpl(() => netServer);
const bser = require('bser');
bunser = new EventEmitter();
bser.BunserBuf.mockImpl(() => bunser);
bser.dumpToBuffer.mockImpl((a) => a);
PackagerServer = require('../../Server');
});
pit('create a server', () => {
const server = new SocketServer('/sock', { projectRoots: ['/root'] });
netServer.emit('listening');
return server.onReady().then(s => {
expect(s).toBe(server);
expect(netServer.listen).toBeCalledWith('/sock');
});
});
pit('handles getDependencies message', () => {
const server = new SocketServer('/sock', { projectRoots: ['/root'] });
netServer.emit('listening');
return server.onReady().then(() => {
const sock = { on: jest.genMockFn(), write: jest.genMockFn() };
netServer.emit('connection', sock);
PackagerServer.prototype.getDependencies.mockImpl(
() => Promise.resolve({ dependencies: ['a', 'b', 'c'] })
);
bunser.emit('value', { type: 'getDependencies', id: 1, data: '/main' });
expect(PackagerServer.prototype.getDependencies).toBeCalledWith('/main');
// Run pending promises.
return Promise.resolve().then(() => {
expect(sock.write).toBeCalledWith(
{ id: 1, type: 'result', data: ['a', 'b', 'c']}
);
});
});
});
pit('handles buildBundle message', () => {
const server = new SocketServer('/sock', { projectRoots: ['/root'] });
netServer.emit('listening');
return server.onReady().then(() => {
const sock = { on: jest.genMockFn(), write: jest.genMockFn() };
netServer.emit('connection', sock);
PackagerServer.prototype.buildBundle.mockImpl(
() => Promise.resolve({ bundle: 'foo' })
);
bunser.emit(
'value',
{ type: 'buildBundle', id: 1, data: { options: 'bar' } }
);
expect(PackagerServer.prototype.buildBundle).toBeCalledWith(
{ options: 'bar' }
);
// Run pending promises.
return Promise.resolve().then(() => {
expect(sock.write).toBeCalledWith(
{ id: 1, type: 'result', data: { bundle: 'foo' }}
);
});
});
});
});

View File

@ -0,0 +1,102 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
const Promise = require('promise');
const SocketClient = require('./SocketClient');
const SocketServer = require('./SocketServer');
const _ = require('underscore');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const tmpdir = require('os').tmpdir();
const {spawn} = require('child_process');
const CREATE_SERVER_TIMEOUT = 10000;
const SocketInterface = {
getOrCreateSocketFor(options) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash('md5');
Object.keys(options).sort().forEach(key => {
if (options[key] && typeof options[key] !== 'string') {
hash.update(JSON.stringify(options[key]));
} else {
hash.update(options[key]);
}
});
const sockPath = path.join(
tmpdir,
'react-packager-' + hash.digest('hex')
);
if (fs.existsSync(sockPath)) {
resolve(SocketClient.create(sockPath));
return;
}
const logPath = path.join(tmpdir, 'react-packager.log');
const timeout = setTimeout(
() => reject(
new Error(
'Took too long to start server. Server logs: \n' +
fs.readFileSync(logPath, 'utf8')
)
),
CREATE_SERVER_TIMEOUT,
);
const log = fs.openSync(logPath, 'a');
// Enable server debugging by default since it's going to a log file.
const env = _.clone(process.env);
env.DEBUG = 'ReactPackager:SocketServer';
// We have to go through the main entry point to make sure
// we go through the babel require hook.
const child = spawn(
process.execPath,
[path.join(__dirname, '..', '..', 'index.js')],
{
detached: true,
env: env,
stdio: ['ipc', log, log]
}
);
child.unref();
child.on('message', m => {
if (m && m.type && m.type === 'createdServer') {
clearTimeout(timeout);
child.disconnect();
resolve(SocketClient.create(sockPath));
}
});
if (options.blacklistRE) {
options.blacklistRE = { source: options.blacklistRE.source };
}
child.send({
type: 'createSocketServer',
data: { sockPath, options }
});
});
},
createSocketServer(sockPath, options) {
return new SocketServer(sockPath, options).onReady();
}
};
module.exports = SocketInterface;